2024-11-07 15:53:11,641 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 15:53:11,656 main DEBUG Took 0.012602 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 15:53:11,656 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 15:53:11,657 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 15:53:11,657 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 15:53:11,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,668 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 15:53:11,680 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,682 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,683 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,684 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,686 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,687 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,688 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,689 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,690 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,691 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,692 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,693 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,694 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 15:53:11,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,696 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 15:53:11,698 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 15:53:11,700 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 15:53:11,703 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 15:53:11,703 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 15:53:11,705 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 15:53:11,705 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 15:53:11,717 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 15:53:11,720 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 15:53:11,723 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 15:53:11,723 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 15:53:11,724 main DEBUG createAppenders(={Console}) 2024-11-07 15:53:11,724 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-07 15:53:11,725 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-07 15:53:11,725 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-07 15:53:11,726 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 15:53:11,726 main DEBUG OutputStream closed 2024-11-07 15:53:11,727 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 15:53:11,727 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 15:53:11,728 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-07 15:53:11,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 15:53:11,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 15:53:11,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 15:53:11,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 15:53:11,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 15:53:11,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 15:53:11,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 15:53:11,812 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 15:53:11,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 15:53:11,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 15:53:11,813 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 15:53:11,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 15:53:11,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 15:53:11,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 15:53:11,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 15:53:11,815 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 15:53:11,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 15:53:11,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 15:53:11,819 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 15:53:11,819 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-07 15:53:11,819 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 15:53:11,820 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-07T15:53:12,086 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a 2024-11-07 15:53:12,089 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 15:53:12,089 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T15:53:12,100 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-07T15:53:12,126 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T15:53:12,142 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9, deleteOnExit=true 2024-11-07T15:53:12,143 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T15:53:12,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/test.cache.data in system properties and HBase conf 2024-11-07T15:53:12,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T15:53:12,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.log.dir in system properties and HBase conf 2024-11-07T15:53:12,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T15:53:12,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T15:53:12,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T15:53:12,239 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T15:53:12,330 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T15:53:12,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T15:53:12,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T15:53:12,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T15:53:12,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T15:53:12,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T15:53:12,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T15:53:12,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T15:53:12,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T15:53:12,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T15:53:12,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/nfs.dump.dir in system properties and HBase conf 2024-11-07T15:53:12,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/java.io.tmpdir in system properties and HBase conf 2024-11-07T15:53:12,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T15:53:12,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T15:53:12,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T15:53:13,187 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T15:53:13,276 INFO [Time-limited test {}] log.Log(170): Logging initialized @2409ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T15:53:13,362 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:53:13,432 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T15:53:13,457 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T15:53:13,457 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T15:53:13,459 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T15:53:13,471 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:53:13,474 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.log.dir/,AVAILABLE} 2024-11-07T15:53:13,475 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T15:53:13,684 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/java.io.tmpdir/jetty-localhost-39653-hadoop-hdfs-3_4_1-tests_jar-_-any-10122042921941257795/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T15:53:13,691 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39653} 2024-11-07T15:53:13,692 INFO [Time-limited test {}] server.Server(415): Started @2826ms 2024-11-07T15:53:14,094 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T15:53:14,102 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T15:53:14,104 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T15:53:14,104 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T15:53:14,104 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-07T15:53:14,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.log.dir/,AVAILABLE} 2024-11-07T15:53:14,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T15:53:14,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/java.io.tmpdir/jetty-localhost-43855-hadoop-hdfs-3_4_1-tests_jar-_-any-11988259547379842280/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T15:53:14,234 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:43855} 2024-11-07T15:53:14,234 INFO [Time-limited test {}] server.Server(415): Started @3369ms 2024-11-07T15:53:14,289 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T15:53:14,733 WARN [Thread-70 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data2/current/BP-480826019-172.17.0.2-1730994792941/current, will proceed with Du for space computation calculation, 2024-11-07T15:53:14,733 WARN [Thread-69 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data1/current/BP-480826019-172.17.0.2-1730994792941/current, will proceed with Du for space computation calculation, 2024-11-07T15:53:14,778 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T15:53:14,838 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dc22892f680e33c with lease ID 0xfef1b25612189e3a: Processing first storage report for DS-b2d167b3-4fad-4a95-8328-e4aebf691dbb from datanode DatanodeRegistration(127.0.0.1:45323, datanodeUuid=4cd907ef-b693-4f91-9ddc-5a993d1d9072, infoPort=40817, infoSecurePort=0, ipcPort=44793, storageInfo=lv=-57;cid=testClusterID;nsid=2073329130;c=1730994792941) 2024-11-07T15:53:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dc22892f680e33c with lease ID 0xfef1b25612189e3a: from storage DS-b2d167b3-4fad-4a95-8328-e4aebf691dbb node DatanodeRegistration(127.0.0.1:45323, datanodeUuid=4cd907ef-b693-4f91-9ddc-5a993d1d9072, infoPort=40817, infoSecurePort=0, ipcPort=44793, storageInfo=lv=-57;cid=testClusterID;nsid=2073329130;c=1730994792941), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T15:53:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9dc22892f680e33c with lease ID 0xfef1b25612189e3a: Processing first storage report for DS-c90aaea0-aa26-43d3-b554-8656d301502f from datanode DatanodeRegistration(127.0.0.1:45323, datanodeUuid=4cd907ef-b693-4f91-9ddc-5a993d1d9072, infoPort=40817, infoSecurePort=0, ipcPort=44793, storageInfo=lv=-57;cid=testClusterID;nsid=2073329130;c=1730994792941) 2024-11-07T15:53:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9dc22892f680e33c with lease ID 0xfef1b25612189e3a: from storage DS-c90aaea0-aa26-43d3-b554-8656d301502f node DatanodeRegistration(127.0.0.1:45323, datanodeUuid=4cd907ef-b693-4f91-9ddc-5a993d1d9072, infoPort=40817, infoSecurePort=0, ipcPort=44793, storageInfo=lv=-57;cid=testClusterID;nsid=2073329130;c=1730994792941), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T15:53:14,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a 2024-11-07T15:53:15,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/zookeeper_0, clientPort=49390, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T15:53:15,025 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49390 2024-11-07T15:53:15,041 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:15,046 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:15,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741825_1001 (size=7) 2024-11-07T15:53:15,695 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 with version=8 2024-11-07T15:53:15,696 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/hbase-staging 2024-11-07T15:53:15,799 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T15:53:16,058 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2682f91c1e2:0 server-side Connection retries=45 2024-11-07T15:53:16,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,075 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T15:53:16,076 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,076 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T15:53:16,234 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T15:53:16,298 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T15:53:16,307 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T15:53:16,311 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T15:53:16,340 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 29512 (auto-detected) 2024-11-07T15:53:16,342 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T15:53:16,367 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39943 2024-11-07T15:53:16,391 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39943 connecting to ZooKeeper ensemble=127.0.0.1:49390 2024-11-07T15:53:16,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399430x0, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T15:53:16,427 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39943-0x101836fe7560000 connected 2024-11-07T15:53:16,461 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:16,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:16,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:53:16,480 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059, hbase.cluster.distributed=false 2024-11-07T15:53:16,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T15:53:16,513 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39943 2024-11-07T15:53:16,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39943 2024-11-07T15:53:16,514 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39943 2024-11-07T15:53:16,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39943 2024-11-07T15:53:16,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39943 2024-11-07T15:53:16,647 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2682f91c1e2:0 server-side Connection retries=45 2024-11-07T15:53:16,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,650 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T15:53:16,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T15:53:16,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T15:53:16,654 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T15:53:16,656 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T15:53:16,657 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42309 2024-11-07T15:53:16,660 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42309 connecting to ZooKeeper ensemble=127.0.0.1:49390 2024-11-07T15:53:16,662 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:16,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:16,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:423090x0, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T15:53:16,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:423090x0, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:53:16,678 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42309-0x101836fe7560001 connected 2024-11-07T15:53:16,683 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T15:53:16,693 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T15:53:16,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T15:53:16,702 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T15:53:16,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42309 2024-11-07T15:53:16,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42309 2024-11-07T15:53:16,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42309 2024-11-07T15:53:16,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42309 2024-11-07T15:53:16,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42309 2024-11-07T15:53:16,726 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2682f91c1e2:39943 2024-11-07T15:53:16,727 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:16,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:53:16,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:53:16,737 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:16,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:16,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T15:53:16,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:16,760 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T15:53:16,762 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2682f91c1e2,39943,1730994795853 from backup master directory 2024-11-07T15:53:16,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:16,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:53:16,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T15:53:16,768 WARN [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T15:53:16,769 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:16,771 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T15:53:16,772 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T15:53:16,838 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/hbase.id] with ID: c6c0d170-2695-406e-8dc6-9b9c44c39fbc 2024-11-07T15:53:16,839 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/.tmp/hbase.id 2024-11-07T15:53:16,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741826_1002 (size=42) 2024-11-07T15:53:16,851 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/.tmp/hbase.id]:[hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/hbase.id] 2024-11-07T15:53:16,901 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T15:53:16,906 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T15:53:16,926 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-07T15:53:16,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:16,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:16,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741827_1003 (size=196) 2024-11-07T15:53:16,963 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:53:16,966 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T15:53:16,985 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:16,989 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:53:17,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741828_1004 (size=1189) 2024-11-07T15:53:17,040 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store 2024-11-07T15:53:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741829_1005 (size=34) 2024-11-07T15:53:17,468 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T15:53:17,472 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:17,473 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T15:53:17,473 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:53:17,473 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:53:17,475 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-07T15:53:17,475 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:53:17,475 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:53:17,476 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730994797473Disabling compacts and flushes for region at 1730994797473Disabling writes for close at 1730994797475 (+2 ms)Writing region close event to WAL at 1730994797475Closed at 1730994797475 2024-11-07T15:53:17,479 WARN [master/c2682f91c1e2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/.initializing 2024-11-07T15:53:17,479 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/WALs/c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:17,488 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T15:53:17,507 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2682f91c1e2%2C39943%2C1730994795853, suffix=, logDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/WALs/c2682f91c1e2,39943,1730994795853, archiveDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/oldWALs, maxLogs=10 2024-11-07T15:53:17,535 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/WALs/c2682f91c1e2,39943,1730994795853/c2682f91c1e2%2C39943%2C1730994795853.1730994797513, exclude list is [], retry=0 2024-11-07T15:53:17,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45323,DS-b2d167b3-4fad-4a95-8328-e4aebf691dbb,DISK] 2024-11-07T15:53:17,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T15:53:17,600 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/WALs/c2682f91c1e2,39943,1730994795853/c2682f91c1e2%2C39943%2C1730994795853.1730994797513 2024-11-07T15:53:17,602 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40817:40817)] 2024-11-07T15:53:17,602 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:53:17,603 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:17,606 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,607 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T15:53:17,678 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:17,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:17,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T15:53:17,685 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:17,686 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:17,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,689 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T15:53:17,689 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:17,690 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:17,691 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T15:53:17,693 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:17,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:17,695 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,698 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,700 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,706 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,706 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,711 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T15:53:17,714 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T15:53:17,719 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:53:17,720 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64933662, jitterRate=-0.03241303563117981}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T15:53:17,726 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730994797619Initializing all the Stores at 1730994797621 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994797622 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994797622Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994797623 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994797623Cleaning up temporary data from old regions at 1730994797707 (+84 ms)Region opened successfully at 1730994797726 (+19 ms) 2024-11-07T15:53:17,727 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T15:53:17,763 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30baf26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2682f91c1e2/172.17.0.2:0 2024-11-07T15:53:17,795 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T15:53:17,807 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-07T15:53:17,807 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-07T15:53:17,810 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T15:53:17,812 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T15:53:17,817 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T15:53:17,817 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T15:53:17,843 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T15:53:17,852 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T15:53:17,855 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T15:53:17,857 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T15:53:17,858 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T15:53:17,861 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T15:53:17,863 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T15:53:17,867 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T15:53:17,868 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T15:53:17,870 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T15:53:17,872 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T15:53:17,894 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T15:53:17,896 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T15:53:17,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T15:53:17,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T15:53:17,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,903 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2682f91c1e2,39943,1730994795853, sessionid=0x101836fe7560000, setting cluster-up flag (Was=false) 2024-11-07T15:53:17,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,925 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T15:53:17,928 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:17,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:17,942 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T15:53:17,944 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:17,951 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T15:53:18,014 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(746): ClusterId : c6c0d170-2695-406e-8dc6-9b9c44c39fbc 2024-11-07T15:53:18,017 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T15:53:18,022 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T15:53:18,022 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T15:53:18,025 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T15:53:18,026 DEBUG [RS:0;c2682f91c1e2:42309 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e38b1f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2682f91c1e2/172.17.0.2:0 2024-11-07T15:53:18,030 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T15:53:18,041 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T15:53:18,041 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2682f91c1e2:42309 2024-11-07T15:53:18,044 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T15:53:18,044 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T15:53:18,044 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T15:53:18,047 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2682f91c1e2,39943,1730994795853 with port=42309, startcode=1730994796599 2024-11-07T15:53:18,048 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T15:53:18,054 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2682f91c1e2,39943,1730994795853 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T15:53:18,061 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2682f91c1e2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:53:18,061 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2682f91c1e2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:53:18,062 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2682f91c1e2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:53:18,062 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2682f91c1e2:0, corePoolSize=5, maxPoolSize=5 2024-11-07T15:53:18,062 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2682f91c1e2:0, corePoolSize=10, maxPoolSize=10 2024-11-07T15:53:18,062 DEBUG [RS:0;c2682f91c1e2:42309 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T15:53:18,062 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,063 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2682f91c1e2:0, corePoolSize=2, maxPoolSize=2 2024-11-07T15:53:18,063 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,066 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730994828066 2024-11-07T15:53:18,068 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T15:53:18,068 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:53:18,069 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T15:53:18,069 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T15:53:18,074 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T15:53:18,074 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T15:53:18,075 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T15:53:18,075 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T15:53:18,075 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,076 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T15:53:18,076 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,079 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T15:53:18,081 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T15:53:18,081 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T15:53:18,088 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T15:53:18,088 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T15:53:18,091 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.large.0-1730994798090,5,FailOnTimeoutGroup] 2024-11-07T15:53:18,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741831_1007 (size=1321) 2024-11-07T15:53:18,091 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.small.0-1730994798091,5,FailOnTimeoutGroup] 2024-11-07T15:53:18,092 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,093 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T15:53:18,094 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T15:53:18,095 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,095 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:53:18,095 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,103 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T15:53:18,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741832_1008 (size=32) 2024-11-07T15:53:18,110 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39943 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,112 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39943 {}] master.ServerManager(517): Registering regionserver=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,127 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:53:18,128 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35131 2024-11-07T15:53:18,128 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T15:53:18,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T15:53:18,134 DEBUG [RS:0;c2682f91c1e2:42309 {}] zookeeper.ZKUtil(111): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,134 WARN [RS:0;c2682f91c1e2:42309 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T15:53:18,134 INFO [RS:0;c2682f91c1e2:42309 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:53:18,135 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,136 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2682f91c1e2,42309,1730994796599] 2024-11-07T15:53:18,168 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T15:53:18,183 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T15:53:18,188 INFO [RS:0;c2682f91c1e2:42309 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T15:53:18,189 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,190 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T15:53:18,196 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T15:53:18,197 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,198 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2682f91c1e2:0, corePoolSize=2, maxPoolSize=2 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,199 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2682f91c1e2:0, corePoolSize=1, maxPoolSize=1 2024-11-07T15:53:18,200 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2682f91c1e2:0, corePoolSize=3, maxPoolSize=3 2024-11-07T15:53:18,200 DEBUG [RS:0;c2682f91c1e2:42309 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0, corePoolSize=3, maxPoolSize=3 2024-11-07T15:53:18,201 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,201 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,201 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,201 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,202 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,202 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,42309,1730994796599-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T15:53:18,222 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T15:53:18,225 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,42309,1730994796599-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,225 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,225 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.Replication(171): c2682f91c1e2,42309,1730994796599 started 2024-11-07T15:53:18,245 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:18,245 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1482): Serving as c2682f91c1e2,42309,1730994796599, RpcServer on c2682f91c1e2/172.17.0.2:42309, sessionid=0x101836fe7560001 2024-11-07T15:53:18,246 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T15:53:18,246 DEBUG [RS:0;c2682f91c1e2:42309 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,247 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2682f91c1e2,42309,1730994796599' 2024-11-07T15:53:18,247 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T15:53:18,248 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T15:53:18,248 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T15:53:18,248 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T15:53:18,249 DEBUG [RS:0;c2682f91c1e2:42309 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,249 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2682f91c1e2,42309,1730994796599' 2024-11-07T15:53:18,249 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T15:53:18,249 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T15:53:18,250 DEBUG [RS:0;c2682f91c1e2:42309 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T15:53:18,250 INFO [RS:0;c2682f91c1e2:42309 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T15:53:18,250 INFO [RS:0;c2682f91c1e2:42309 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T15:53:18,356 INFO [RS:0;c2682f91c1e2:42309 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T15:53:18,359 INFO [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2682f91c1e2%2C42309%2C1730994796599, suffix=, logDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599, archiveDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/oldWALs, maxLogs=32 2024-11-07T15:53:18,376 DEBUG [RS:0;c2682f91c1e2:42309 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599/c2682f91c1e2%2C42309%2C1730994796599.1730994798362, exclude list is [], retry=0 2024-11-07T15:53:18,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45323,DS-b2d167b3-4fad-4a95-8328-e4aebf691dbb,DISK] 2024-11-07T15:53:18,385 INFO [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599/c2682f91c1e2%2C42309%2C1730994796599.1730994798362 2024-11-07T15:53:18,385 DEBUG [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40817:40817)] 2024-11-07T15:53:18,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:18,514 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T15:53:18,517 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T15:53:18,517 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:18,518 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T15:53:18,520 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T15:53:18,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:18,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T15:53:18,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T15:53:18,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:18,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T15:53:18,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T15:53:18,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:18,529 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T15:53:18,530 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740 2024-11-07T15:53:18,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740 2024-11-07T15:53:18,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T15:53:18,534 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T15:53:18,535 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:53:18,537 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T15:53:18,542 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:53:18,543 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64651915, jitterRate=-0.036611393094062805}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:53:18,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730994798511Initializing all the Stores at 1730994798514 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798514Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798514Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994798514Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798514Cleaning up temporary data from old regions at 1730994798534 (+20 ms)Region opened successfully at 1730994798545 (+11 ms) 2024-11-07T15:53:18,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T15:53:18,545 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T15:53:18,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T15:53:18,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T15:53:18,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T15:53:18,547 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T15:53:18,547 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730994798545Disabling compacts and flushes for region at 1730994798545Disabling writes for close at 1730994798545Writing region close event to WAL at 1730994798546 (+1 ms)Closed at 1730994798547 (+1 ms) 2024-11-07T15:53:18,551 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:53:18,551 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T15:53:18,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T15:53:18,566 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T15:53:18,569 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T15:53:18,722 DEBUG [c2682f91c1e2:39943 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-07T15:53:18,734 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:18,742 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2682f91c1e2,42309,1730994796599, state=OPENING 2024-11-07T15:53:18,748 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T15:53:18,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:18,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:53:18,751 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:53:18,751 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:53:18,753 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T15:53:18,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:18,931 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T15:53:18,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T15:53:18,946 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T15:53:18,946 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T15:53:18,947 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T15:53:18,951 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2682f91c1e2%2C42309%2C1730994796599.meta, suffix=.meta, logDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599, archiveDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/oldWALs, maxLogs=32 2024-11-07T15:53:18,968 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599/c2682f91c1e2%2C42309%2C1730994796599.meta.1730994798953.meta, exclude list is [], retry=0 2024-11-07T15:53:18,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45323,DS-b2d167b3-4fad-4a95-8328-e4aebf691dbb,DISK] 2024-11-07T15:53:18,976 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/WALs/c2682f91c1e2,42309,1730994796599/c2682f91c1e2%2C42309%2C1730994796599.meta.1730994798953.meta 2024-11-07T15:53:18,976 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40817:40817)] 2024-11-07T15:53:18,977 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:53:18,978 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T15:53:18,981 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T15:53:18,986 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T15:53:18,990 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T15:53:18,991 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:18,991 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T15:53:18,991 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T15:53:18,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T15:53:18,996 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T15:53:18,996 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:18,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:18,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T15:53:18,999 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T15:53:18,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:19,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:19,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T15:53:19,001 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T15:53:19,001 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:19,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:19,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T15:53:19,004 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T15:53:19,004 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:19,005 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T15:53:19,005 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T15:53:19,006 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740 2024-11-07T15:53:19,009 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740 2024-11-07T15:53:19,012 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T15:53:19,012 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T15:53:19,013 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:53:19,015 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T15:53:19,017 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71197152, jitterRate=0.06092023849487305}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:53:19,017 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T15:53:19,019 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730994798992Writing region info on filesystem at 1730994798992Initializing all the Stores at 1730994798994 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798994Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798994Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994798994Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730994798994Cleaning up temporary data from old regions at 1730994799012 (+18 ms)Running coprocessor post-open hooks at 1730994799017 (+5 ms)Region opened successfully at 1730994799019 (+2 ms) 2024-11-07T15:53:19,027 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730994798922 2024-11-07T15:53:19,038 DEBUG [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T15:53:19,039 INFO [RS_OPEN_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T15:53:19,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:19,043 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2682f91c1e2,42309,1730994796599, state=OPEN 2024-11-07T15:53:19,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T15:53:19,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T15:53:19,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:53:19,049 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T15:53:19,050 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:19,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T15:53:19,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2682f91c1e2,42309,1730994796599 in 296 msec 2024-11-07T15:53:19,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T15:53:19,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 501 msec 2024-11-07T15:53:19,064 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T15:53:19,064 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T15:53:19,085 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:19,087 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:19,116 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41195, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:19,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1670 sec 2024-11-07T15:53:19,148 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730994799148, completionTime=-1 2024-11-07T15:53:19,152 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-07T15:53:19,152 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T15:53:19,178 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-07T15:53:19,178 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730994859178 2024-11-07T15:53:19,178 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730994919178 2024-11-07T15:53:19,178 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-07T15:53:19,181 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,182 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,182 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,184 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2682f91c1e2:39943, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,184 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,185 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,190 DEBUG [master/c2682f91c1e2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T15:53:19,212 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.443sec 2024-11-07T15:53:19,214 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T15:53:19,243 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T15:53:19,244 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T15:53:19,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:19,245 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T15:53:19,245 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T15:53:19,246 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T15:53:19,247 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T15:53:19,248 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T15:53:19,248 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T15:53:19,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:19,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:19,257 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T15:53:19,261 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T15:53:19,262 INFO [master/c2682f91c1e2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2682f91c1e2,39943,1730994795853-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T15:53:19,266 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dc9c1f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:19,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:19,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:19,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:19,277 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:19,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:19,280 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:19,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:19,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:19,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:19,321 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=210, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=120, ProcessCount=11, AvailableMemoryMB=6293 2024-11-07T15:53:19,330 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:53:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:19,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@22f4e74d 2024-11-07T15:53:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:53:19,341 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54394, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:53:19,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:53:19,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:53:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:19,357 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:53:19,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-07T15:53:19,359 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:19,361 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:53:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:19,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741835_1011 (size=959) 2024-11-07T15:53:19,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:19,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:19,776 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:53:19,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741836_1012 (size=53) 2024-11-07T15:53:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:20,185 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:20,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b2da43428956e53a96aa3524187a4a97, disabling compactions & flushes 2024-11-07T15:53:20,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. after waiting 0 ms 2024-11-07T15:53:20,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,186 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,186 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b2da43428956e53a96aa3524187a4a97: Waiting for close lock at 1730994800186Disabling compacts and flushes for region at 1730994800186Disabling writes for close at 1730994800186Writing region close event to WAL at 1730994800186Closed at 1730994800186 2024-11-07T15:53:20,189 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:53:20,194 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994800189"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994800189"}]},"ts":"1730994800189"} 2024-11-07T15:53:20,200 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:53:20,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:53:20,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994800202"}]},"ts":"1730994800202"} 2024-11-07T15:53:20,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:53:20,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, ASSIGN}] 2024-11-07T15:53:20,213 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, ASSIGN 2024-11-07T15:53:20,215 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:53:20,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2da43428956e53a96aa3524187a4a97, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:20,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, ASSIGN because future has completed 2024-11-07T15:53:20,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:20,534 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,534 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:53:20,535 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,535 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:20,535 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,535 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,538 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,541 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2da43428956e53a96aa3524187a4a97 columnFamilyName A 2024-11-07T15:53:20,541 DEBUG [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:20,542 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(327): Store=b2da43428956e53a96aa3524187a4a97/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:20,542 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,545 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2da43428956e53a96aa3524187a4a97 columnFamilyName B 2024-11-07T15:53:20,545 DEBUG [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:20,546 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(327): Store=b2da43428956e53a96aa3524187a4a97/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:20,546 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,548 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2da43428956e53a96aa3524187a4a97 columnFamilyName C 2024-11-07T15:53:20,548 DEBUG [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:20,549 INFO [StoreOpener-b2da43428956e53a96aa3524187a4a97-1 {}] regionserver.HStore(327): Store=b2da43428956e53a96aa3524187a4a97/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:20,549 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,550 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,551 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,553 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,553 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,554 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:53:20,556 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,559 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:53:20,560 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b2da43428956e53a96aa3524187a4a97; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72098691, jitterRate=0.07435421645641327}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:53:20,560 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:20,561 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b2da43428956e53a96aa3524187a4a97: Running coprocessor pre-open hook at 1730994800536Writing region info on filesystem at 1730994800536Initializing all the Stores at 1730994800538 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994800538Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994800538Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994800538Cleaning up temporary data from old regions at 1730994800553 (+15 ms)Running coprocessor post-open hooks at 1730994800560 (+7 ms)Region opened successfully at 1730994800561 (+1 ms) 2024-11-07T15:53:20,564 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., pid=6, masterSystemTime=1730994800527 2024-11-07T15:53:20,567 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,567 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:20,568 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2da43428956e53a96aa3524187a4a97, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:20,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:53:20,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-07T15:53:20,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 in 202 msec 2024-11-07T15:53:20,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-07T15:53:20,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, ASSIGN in 367 msec 2024-11-07T15:53:20,583 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:53:20,583 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994800583"}]},"ts":"1730994800583"} 2024-11-07T15:53:20,586 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:53:20,588 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:53:20,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2390 sec 2024-11-07T15:53:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T15:53:21,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:21,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8feb56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,524 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db46afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,527 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66225631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,529 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,534 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,536 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:53:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6b9abfc3 2024-11-07T15:53:21,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:53:21,541 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:53:21,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6432e8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ace4bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c2c6177, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@392f723b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,555 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2311693d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,558 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ce3c82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0e98fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,565 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69066494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2da332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,573 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f1c733d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,577 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b782e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,579 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32310937, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ec8db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,583 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108826c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c382bba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,590 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,590 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd5f624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,594 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b18d6c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef2ef1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,600 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:21,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:21,606 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:21,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b76b53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:21,608 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dadec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:21,617 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,617 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,620 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,622 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,630 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,631 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,635 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-07T15:53:21,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:21,644 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:21,644 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:21,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:21,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,666 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,677 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,684 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:21,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:21,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:53:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:21,743 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:21,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:53:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:21,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:53:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:21,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:21,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:21,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:21,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:21,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5b512bf9b0de415f987179d7873a9cda is 50, key is test_row_0/A:col10/1730994801733/Put/seqid=0 2024-11-07T15:53:21,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:21,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994861884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994861902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994861914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:21,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741837_1013 (size=12001) 2024-11-07T15:53:21,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994861916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:21,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994861919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:21,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5b512bf9b0de415f987179d7873a9cda 2024-11-07T15:53:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:21,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:53:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:22,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2d2313e531df499888ce1574c5cb8e5f is 50, key is test_row_0/B:col10/1730994801733/Put/seqid=0 2024-11-07T15:53:22,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994862074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994862076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994862077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994862078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994862078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741838_1014 (size=12001) 2024-11-07T15:53:22,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2d2313e531df499888ce1574c5cb8e5f 2024-11-07T15:53:22,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/611fd72873044363a120822df3f86f5a is 50, key is test_row_0/C:col10/1730994801733/Put/seqid=0 2024-11-07T15:53:22,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741839_1015 (size=12001) 2024-11-07T15:53:22,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/611fd72873044363a120822df3f86f5a 2024-11-07T15:53:22,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5b512bf9b0de415f987179d7873a9cda as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda 2024-11-07T15:53:22,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:53:22,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:53:22,239 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-07T15:53:22,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2d2313e531df499888ce1574c5cb8e5f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f 2024-11-07T15:53:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:53:22,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/611fd72873044363a120822df3f86f5a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a 2024-11-07T15:53:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:22,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:53:22,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for b2da43428956e53a96aa3524187a4a97 in 537ms, sequenceid=16, compaction requested=false 2024-11-07T15:53:22,288 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:53:22,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:22,308 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:22,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:53:22,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/02b0abc544394e81842ca1cbb87d292d is 50, key is test_row_0/A:col10/1730994802300/Put/seqid=0 2024-11-07T15:53:22,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741840_1016 (size=16681) 2024-11-07T15:53:22,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/02b0abc544394e81842ca1cbb87d292d 2024-11-07T15:53:22,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994862339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994862342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994862344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994862347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/5f37346914af4059b12b22168a1145ea is 50, key is test_row_0/B:col10/1730994802300/Put/seqid=0 2024-11-07T15:53:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994862353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741841_1017 (size=12001) 2024-11-07T15:53:22,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/5f37346914af4059b12b22168a1145ea 2024-11-07T15:53:22,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:53:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:22,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:22,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/9888d605d7894bedb0e88efd03a23448 is 50, key is test_row_0/C:col10/1730994802300/Put/seqid=0 2024-11-07T15:53:22,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741842_1018 (size=12001) 2024-11-07T15:53:22,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/9888d605d7894bedb0e88efd03a23448 2024-11-07T15:53:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994862464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994862464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/02b0abc544394e81842ca1cbb87d292d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d 2024-11-07T15:53:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994862474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d, entries=250, sequenceid=40, filesize=16.3 K 2024-11-07T15:53:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994862476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994862476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/5f37346914af4059b12b22168a1145ea as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea 2024-11-07T15:53:22,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:53:22,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/9888d605d7894bedb0e88efd03a23448 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448 2024-11-07T15:53:22,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:53:22,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b2da43428956e53a96aa3524187a4a97 in 219ms, sequenceid=40, compaction requested=false 2024-11-07T15:53:22,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:22,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-07T15:53:22,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,564 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:53:22,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/7de29799a0b341069682da18fbe09268 is 50, key is test_row_0/A:col10/1730994802337/Put/seqid=0 2024-11-07T15:53:22,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741843_1019 (size=12001) 2024-11-07T15:53:22,600 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/7de29799a0b341069682da18fbe09268 2024-11-07T15:53:22,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3d6c31a762ac4c5c961d6fd9deaea4f5 is 50, key is test_row_0/B:col10/1730994802337/Put/seqid=0 2024-11-07T15:53:22,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741844_1020 (size=12001) 2024-11-07T15:53:22,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3d6c31a762ac4c5c961d6fd9deaea4f5 2024-11-07T15:53:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1358d18a56364cf38d7138a3be1d988d is 50, key is test_row_0/C:col10/1730994802337/Put/seqid=0 2024-11-07T15:53:22,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:22,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:22,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741845_1021 (size=12001) 2024-11-07T15:53:22,691 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1358d18a56364cf38d7138a3be1d988d 2024-11-07T15:53:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/7de29799a0b341069682da18fbe09268 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268 2024-11-07T15:53:22,731 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T15:53:22,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3d6c31a762ac4c5c961d6fd9deaea4f5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5 2024-11-07T15:53:22,750 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T15:53:22,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1358d18a56364cf38d7138a3be1d988d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d 2024-11-07T15:53:22,772 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-07T15:53:22,774 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=107.34 KB/109920 for b2da43428956e53a96aa3524187a4a97 in 210ms, sequenceid=52, compaction requested=true 2024-11-07T15:53:22,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:22,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:22,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-07T15:53:22,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-07T15:53:22,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:22,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:22,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:53:22,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:22,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-07T15:53:22,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1280 sec 2024-11-07T15:53:22,793 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.1580 sec 2024-11-07T15:53:22,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5f449e9dfa7c4a0cbf94e591a9191fe7 is 50, key is test_row_0/A:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:22,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741846_1022 (size=12001) 2024-11-07T15:53:22,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5f449e9dfa7c4a0cbf94e591a9191fe7 2024-11-07T15:53:22,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994862819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994862822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994862829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994862832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,847 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994862832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/d96993b71c6c474285fea209d3e9464d is 50, key is test_row_0/B:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:22,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741847_1023 (size=12001) 2024-11-07T15:53:22,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/d96993b71c6c474285fea209d3e9464d 2024-11-07T15:53:22,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5a8c40ffb1c04c40beb4da80f2e767b2 is 50, key is test_row_0/C:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:22,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741848_1024 (size=12001) 2024-11-07T15:53:22,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5a8c40ffb1c04c40beb4da80f2e767b2 2024-11-07T15:53:22,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5f449e9dfa7c4a0cbf94e591a9191fe7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7 2024-11-07T15:53:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994862945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994862947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994862955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994862955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994862955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:22,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T15:53:22,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/d96993b71c6c474285fea209d3e9464d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d 2024-11-07T15:53:22,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T15:53:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5a8c40ffb1c04c40beb4da80f2e767b2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2 2024-11-07T15:53:23,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2, entries=150, sequenceid=73, filesize=11.7 K 2024-11-07T15:53:23,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for b2da43428956e53a96aa3524187a4a97 in 229ms, sequenceid=73, compaction requested=true 2024-11-07T15:53:23,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:23,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:23,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:23,030 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:23,030 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:23,035 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:23,037 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:23,039 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:23,039 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:23,040 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=46.9 K 2024-11-07T15:53:23,042 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2313e531df499888ce1574c5cb8e5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994801733 2024-11-07T15:53:23,042 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:23,042 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:23,042 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=51.4 K 2024-11-07T15:53:23,043 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f37346914af4059b12b22168a1145ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994801910 2024-11-07T15:53:23,043 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b512bf9b0de415f987179d7873a9cda, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994801733 2024-11-07T15:53:23,044 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6c31a762ac4c5c961d6fd9deaea4f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730994802325 2024-11-07T15:53:23,044 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02b0abc544394e81842ca1cbb87d292d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994801897 2024-11-07T15:53:23,045 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de29799a0b341069682da18fbe09268, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730994802325 2024-11-07T15:53:23,045 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d96993b71c6c474285fea209d3e9464d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:23,046 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f449e9dfa7c4a0cbf94e591a9191fe7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:23,111 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#13 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:23,111 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#12 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:23,113 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c934c91ab5544dd5a61764799a8aa4c7 is 50, key is test_row_0/B:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:23,113 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/7cf2aefe184b4f7ca693bfe0548847e8 is 50, key is test_row_0/A:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:23,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741849_1025 (size=12139) 2024-11-07T15:53:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741850_1026 (size=12139) 2024-11-07T15:53:23,179 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/7cf2aefe184b4f7ca693bfe0548847e8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7cf2aefe184b4f7ca693bfe0548847e8 2024-11-07T15:53:23,183 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c934c91ab5544dd5a61764799a8aa4c7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c934c91ab5544dd5a61764799a8aa4c7 2024-11-07T15:53:23,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:23,186 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:23,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:53:23,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a55f244d93e041b289b82cdd60ae84c7 is 50, key is test_row_0/A:col10/1730994802827/Put/seqid=0 2024-11-07T15:53:23,211 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into c934c91ab5544dd5a61764799a8aa4c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:23,211 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 7cf2aefe184b4f7ca693bfe0548847e8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:23,211 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:23,212 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:23,214 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=12, startTime=1730994803027; duration=0sec 2024-11-07T15:53:23,215 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:23,217 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=12, startTime=1730994803021; duration=0sec 2024-11-07T15:53:23,217 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:23,217 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:23,217 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:23,217 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:23,223 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:23,223 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:23,223 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:23,223 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=46.9 K 2024-11-07T15:53:23,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 611fd72873044363a120822df3f86f5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994801733 2024-11-07T15:53:23,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9888d605d7894bedb0e88efd03a23448, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994801910 2024-11-07T15:53:23,226 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1358d18a56364cf38d7138a3be1d988d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1730994802325 2024-11-07T15:53:23,227 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8c40ffb1c04c40beb4da80f2e767b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:23,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741851_1027 (size=16681) 2024-11-07T15:53:23,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994863238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994863241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994863244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994863248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994863251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,268 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#15 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:23,269 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/dc8c6d3f6647417eb80a8901e46e8336 is 50, key is test_row_0/C:col10/1730994802760/Put/seqid=0 2024-11-07T15:53:23,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741852_1028 (size=12139) 2024-11-07T15:53:23,314 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/dc8c6d3f6647417eb80a8901e46e8336 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/dc8c6d3f6647417eb80a8901e46e8336 2024-11-07T15:53:23,331 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into dc8c6d3f6647417eb80a8901e46e8336(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:23,331 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:23,331 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=12, startTime=1730994803027; duration=0sec 2024-11-07T15:53:23,331 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:23,331 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:23,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994863364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994863366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994863366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994863366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994863366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994863574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994863574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994863584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994863584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994863585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a55f244d93e041b289b82cdd60ae84c7 2024-11-07T15:53:23,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/ffc8feebcbde4ecdbad23e1c5250fa15 is 50, key is test_row_0/B:col10/1730994802827/Put/seqid=0 2024-11-07T15:53:23,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741853_1029 (size=12001) 2024-11-07T15:53:23,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/ffc8feebcbde4ecdbad23e1c5250fa15 2024-11-07T15:53:23,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c387e3855544e2da05f6cae61eb5690 is 50, key is test_row_0/C:col10/1730994802827/Put/seqid=0 2024-11-07T15:53:23,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741854_1030 (size=12001) 2024-11-07T15:53:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-07T15:53:23,794 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:23,795 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-07T15:53:23,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:53:23,804 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:23,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:23,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994863884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994863884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,890 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994863904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994863904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,908 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994863904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:23,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:53:23,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T15:53:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:23,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:23,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:23,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:24,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T15:53:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:24,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:24,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:24,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:53:24,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c387e3855544e2da05f6cae61eb5690 2024-11-07T15:53:24,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a55f244d93e041b289b82cdd60ae84c7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7 2024-11-07T15:53:24,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7, entries=250, sequenceid=91, filesize=16.3 K 2024-11-07T15:53:24,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/ffc8feebcbde4ecdbad23e1c5250fa15 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15 2024-11-07T15:53:24,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T15:53:24,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c387e3855544e2da05f6cae61eb5690 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690 2024-11-07T15:53:24,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T15:53:24,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b2da43428956e53a96aa3524187a4a97 in 1015ms, sequenceid=91, compaction requested=false 2024-11-07T15:53:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:24,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-07T15:53:24,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:24,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:53:24,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/cc75c8b831644738998a4563c7454857 is 50, key is test_row_0/A:col10/1730994803234/Put/seqid=0 2024-11-07T15:53:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741855_1031 (size=12001) 2024-11-07T15:53:24,315 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/cc75c8b831644738998a4563c7454857 2024-11-07T15:53:24,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/10e9465fed914b75ad28091507bd3935 is 50, key is test_row_0/B:col10/1730994803234/Put/seqid=0 2024-11-07T15:53:24,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741856_1032 (size=12001) 2024-11-07T15:53:24,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/10e9465fed914b75ad28091507bd3935 2024-11-07T15:53:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/4003694ca40a4779893b58e2c8b50c63 is 50, key is test_row_0/C:col10/1730994803234/Put/seqid=0 2024-11-07T15:53:24,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:24,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741857_1033 (size=12001) 2024-11-07T15:53:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:53:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994864441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994864445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994864445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994864449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994864450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,503 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:53:24,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994864554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994864554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994864563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994864564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994864564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T15:53:24,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994864764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994864764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994864773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994864773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994864773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:24,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:24,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/4003694ca40a4779893b58e2c8b50c63 2024-11-07T15:53:24,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/cc75c8b831644738998a4563c7454857 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857 2024-11-07T15:53:24,858 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:53:24,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/10e9465fed914b75ad28091507bd3935 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935 2024-11-07T15:53:24,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:53:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/4003694ca40a4779893b58e2c8b50c63 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63 2024-11-07T15:53:24,885 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:53:24,889 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b2da43428956e53a96aa3524187a4a97 in 613ms, sequenceid=112, compaction requested=true 2024-11-07T15:53:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:24,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-07T15:53:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-07T15:53:24,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-07T15:53:24,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0890 sec 2024-11-07T15:53:24,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.1030 sec 2024-11-07T15:53:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-07T15:53:24,944 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:24,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-07T15:53:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:53:24,951 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:24,953 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:24,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:25,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:53:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:25,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:25,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:53:25,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6e3af31186744ddb99b841c359fd4066 is 50, key is test_row_0/A:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741858_1034 (size=16831) 2024-11-07T15:53:25,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6e3af31186744ddb99b841c359fd4066 2024-11-07T15:53:25,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:53:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/b5aa2c439569476bb3e2323005e3f9ee is 50, key is test_row_0/B:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994865118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994865122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994865130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994865122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741859_1035 (size=12151) 2024-11-07T15:53:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994865128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/b5aa2c439569476bb3e2323005e3f9ee 2024-11-07T15:53:25,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/106f77a614fb4dd3a4a07297c57b3ab8 is 50, key is test_row_0/C:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741860_1036 (size=12151) 2024-11-07T15:53:25,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/106f77a614fb4dd3a4a07297c57b3ab8 2024-11-07T15:53:25,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6e3af31186744ddb99b841c359fd4066 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066 2024-11-07T15:53:25,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066, entries=250, sequenceid=132, filesize=16.4 K 2024-11-07T15:53:25,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/b5aa2c439569476bb3e2323005e3f9ee as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee 2024-11-07T15:53:25,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee, entries=150, sequenceid=132, filesize=11.9 K 2024-11-07T15:53:25,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/106f77a614fb4dd3a4a07297c57b3ab8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8 2024-11-07T15:53:25,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8, entries=150, sequenceid=132, filesize=11.9 K 2024-11-07T15:53:25,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b2da43428956e53a96aa3524187a4a97 in 169ms, sequenceid=132, compaction requested=true 2024-11-07T15:53:25,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:25,248 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:25,248 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:25,248 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:25,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:25,249 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:25,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:25,251 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57652 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:25,251 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:25,252 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,252 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7cf2aefe184b4f7ca693bfe0548847e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=56.3 K 2024-11-07T15:53:25,252 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:25,252 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:25,252 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,252 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c934c91ab5544dd5a61764799a8aa4c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=47.2 K 2024-11-07T15:53:25,253 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cf2aefe184b4f7ca693bfe0548847e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:25,253 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c934c91ab5544dd5a61764799a8aa4c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:25,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:25,254 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a55f244d93e041b289b82cdd60ae84c7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994802807 2024-11-07T15:53:25,254 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ffc8feebcbde4ecdbad23e1c5250fa15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994802807 2024-11-07T15:53:25,255 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc75c8b831644738998a4563c7454857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994803234 2024-11-07T15:53:25,255 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e9465fed914b75ad28091507bd3935, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994803234 2024-11-07T15:53:25,256 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b5aa2c439569476bb3e2323005e3f9ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804446 2024-11-07T15:53:25,256 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3af31186744ddb99b841c359fd4066, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804437 2024-11-07T15:53:25,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:25,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:53:25,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:25,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:53:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:25,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:53:25,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:25,280 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:25,281 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/364ba24a084144ff836a38b37026235d is 50, key is test_row_0/A:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/484d98a8d4144b5cb684dad48c2943fe is 50, key is test_row_0/A:col10/1730994805259/Put/seqid=0 2024-11-07T15:53:25,295 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:25,296 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/4b18b4874d904bfe8728cfab169da2b2 is 50, key is test_row_0/B:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741862_1038 (size=14541) 2024-11-07T15:53:25,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741861_1037 (size=12425) 2024-11-07T15:53:25,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/484d98a8d4144b5cb684dad48c2943fe 2024-11-07T15:53:25,309 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/364ba24a084144ff836a38b37026235d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/364ba24a084144ff836a38b37026235d 2024-11-07T15:53:25,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/86ceab5dd2a740ada91414fd30ff67da is 50, key is test_row_0/B:col10/1730994805259/Put/seqid=0 2024-11-07T15:53:25,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741863_1039 (size=12425) 2024-11-07T15:53:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741864_1040 (size=12151) 2024-11-07T15:53:25,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/86ceab5dd2a740ada91414fd30ff67da 2024-11-07T15:53:25,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b7e31497a46347bba586ba9a1d51c822 is 50, key is test_row_0/C:col10/1730994805259/Put/seqid=0 2024-11-07T15:53:25,343 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 364ba24a084144ff836a38b37026235d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:25,343 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,343 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=12, startTime=1730994805248; duration=0sec 2024-11-07T15:53:25,344 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:25,344 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:25,344 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:25,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741865_1041 (size=12151) 2024-11-07T15:53:25,347 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:25,347 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:25,347 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,348 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/dc8c6d3f6647417eb80a8901e46e8336, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=47.2 K 2024-11-07T15:53:25,348 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc8c6d3f6647417eb80a8901e46e8336, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1730994802753 2024-11-07T15:53:25,352 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c387e3855544e2da05f6cae61eb5690, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994802807 2024-11-07T15:53:25,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b7e31497a46347bba586ba9a1d51c822 2024-11-07T15:53:25,354 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4003694ca40a4779893b58e2c8b50c63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994803234 2024-11-07T15:53:25,355 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106f77a614fb4dd3a4a07297c57b3ab8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804446 2024-11-07T15:53:25,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/484d98a8d4144b5cb684dad48c2943fe as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe 2024-11-07T15:53:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994865361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994865365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994865367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe, entries=200, sequenceid=154, filesize=14.2 K 2024-11-07T15:53:25,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/86ceab5dd2a740ada91414fd30ff67da as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da 2024-11-07T15:53:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994865374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,387 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#29 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:25,388 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/fe7f50bd023c44bcb41ee0407ab5405a is 50, key is test_row_0/C:col10/1730994805074/Put/seqid=0 2024-11-07T15:53:25,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994865379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:53:25,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741866_1042 (size=12425) 2024-11-07T15:53:25,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b7e31497a46347bba586ba9a1d51c822 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822 2024-11-07T15:53:25,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822, entries=150, sequenceid=154, filesize=11.9 K 2024-11-07T15:53:25,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-07T15:53:25,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for b2da43428956e53a96aa3524187a4a97 in 156ms, sequenceid=154, compaction requested=false 2024-11-07T15:53:25,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:53:25,430 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/fe7f50bd023c44bcb41ee0407ab5405a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/fe7f50bd023c44bcb41ee0407ab5405a 2024-11-07T15:53:25,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/068f00e0bf7b4b74895360f60d3a7750 is 50, key is test_row_0/A:col10/1730994805364/Put/seqid=0 2024-11-07T15:53:25,486 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into fe7f50bd023c44bcb41ee0407ab5405a(size=12.1 K), total size for store is 24 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:25,486 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,486 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=12, startTime=1730994805249; duration=0sec 2024-11-07T15:53:25,486 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:25,486 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:25,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:25,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741867_1043 (size=12151) 2024-11-07T15:53:25,497 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/068f00e0bf7b4b74895360f60d3a7750 2024-11-07T15:53:25,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/28517390cec144a59a3ee50e83edfaf5 is 50, key is test_row_0/B:col10/1730994805364/Put/seqid=0 2024-11-07T15:53:25,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994865540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994865537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994865543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994865546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994865548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741868_1044 (size=12151) 2024-11-07T15:53:25,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/28517390cec144a59a3ee50e83edfaf5 2024-11-07T15:53:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:53:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b610b2bd178849a9bed696aea3d1643c is 50, key is test_row_0/C:col10/1730994805364/Put/seqid=0 2024-11-07T15:53:25,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741869_1045 (size=12151) 2024-11-07T15:53:25,599 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b610b2bd178849a9bed696aea3d1643c 2024-11-07T15:53:25,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/068f00e0bf7b4b74895360f60d3a7750 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750 2024-11-07T15:53:25,621 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T15:53:25,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/28517390cec144a59a3ee50e83edfaf5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5 2024-11-07T15:53:25,640 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T15:53:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b610b2bd178849a9bed696aea3d1643c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c 2024-11-07T15:53:25,653 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c, entries=150, sequenceid=170, filesize=11.9 K 2024-11-07T15:53:25,657 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b2da43428956e53a96aa3524187a4a97 in 234ms, sequenceid=170, compaction requested=true 2024-11-07T15:53:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:25,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-07T15:53:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-07T15:53:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:25,660 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:25,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:53:25,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-07T15:53:25,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 707 msec 2024-11-07T15:53:25,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e8d26aef6fb84c86881cb511df4d827f is 50, key is test_row_0/A:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:25,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 720 msec 2024-11-07T15:53:25,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994865684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994865687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994865687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994865689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994865693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741870_1046 (size=14541) 2024-11-07T15:53:25,739 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/4b18b4874d904bfe8728cfab169da2b2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4b18b4874d904bfe8728cfab169da2b2 2024-11-07T15:53:25,750 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 4b18b4874d904bfe8728cfab169da2b2(size=12.1 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:25,750 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:25,750 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=12, startTime=1730994805248; duration=0sec 2024-11-07T15:53:25,751 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:25,751 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:25,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994865804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994865804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994865804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994865805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994865805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:25,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994866013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994866013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994866014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994866014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994866014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-07T15:53:26,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:26,085 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-07T15:53:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:26,088 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:26,091 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:26,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:26,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e8d26aef6fb84c86881cb511df4d827f 2024-11-07T15:53:26,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c9ea60f258db4fbaa3c4fd8689e5c60b is 50, key is test_row_0/B:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:26,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741871_1047 (size=12151) 2024-11-07T15:53:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:26,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T15:53:26,296 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T15:53:26,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:53:26,297 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T15:53:26,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T15:53:26,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T15:53:26,299 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:53:26,299 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T15:53:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994866324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994866324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994866324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994866324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,333 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994866333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c9ea60f258db4fbaa3c4fd8689e5c60b 2024-11-07T15:53:26,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b1b5c955d8a84568a52e273e8830dc55 is 50, key is test_row_0/C:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:26,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741872_1048 (size=12151) 2024-11-07T15:53:26,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:26,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994866843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994866844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994866844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994866845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:26,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994866846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:26,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:26,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:26,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b1b5c955d8a84568a52e273e8830dc55 2024-11-07T15:53:26,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e8d26aef6fb84c86881cb511df4d827f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f 2024-11-07T15:53:26,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f, entries=200, sequenceid=193, filesize=14.2 K 2024-11-07T15:53:26,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c9ea60f258db4fbaa3c4fd8689e5c60b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b 2024-11-07T15:53:26,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:53:26,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b1b5c955d8a84568a52e273e8830dc55 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55 2024-11-07T15:53:26,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55, entries=150, sequenceid=193, filesize=11.9 K 2024-11-07T15:53:26,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for b2da43428956e53a96aa3524187a4a97 in 1336ms, sequenceid=193, compaction requested=true 2024-11-07T15:53:26,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:26,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:26,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:26,996 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:26,997 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:26,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:26,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:26,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:26,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:27,000 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53658 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:27,000 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:27,000 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:27,000 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/364ba24a084144ff836a38b37026235d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=52.4 K 2024-11-07T15:53:27,000 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:27,000 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:27,000 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:27,000 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4b18b4874d904bfe8728cfab169da2b2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=47.7 K 2024-11-07T15:53:27,001 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b18b4874d904bfe8728cfab169da2b2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804446 2024-11-07T15:53:27,002 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 364ba24a084144ff836a38b37026235d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804446 2024-11-07T15:53:27,002 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ceab5dd2a740ada91414fd30ff67da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730994805259 2024-11-07T15:53:27,003 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 484d98a8d4144b5cb684dad48c2943fe, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730994805256 2024-11-07T15:53:27,003 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 28517390cec144a59a3ee50e83edfaf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730994805357 2024-11-07T15:53:27,003 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068f00e0bf7b4b74895360f60d3a7750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730994805357 2024-11-07T15:53:27,004 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8d26aef6fb84c86881cb511df4d827f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805529 2024-11-07T15:53:27,004 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ea60f258db4fbaa3c4fd8689e5c60b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805543 2024-11-07T15:53:27,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-07T15:53:27,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:27,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:53:27,027 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:27,028 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/851d0f1d522c4c3bb2ebe29791c5326b is 50, key is test_row_0/B:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:27,035 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:27,036 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/d944a5fd90c44177a50d53d0dde1d82b is 50, key is test_row_0/A:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:27,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/11af1e2c96224496a234161601883683 is 50, key is test_row_0/A:col10/1730994805686/Put/seqid=0 2024-11-07T15:53:27,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741873_1049 (size=12561) 2024-11-07T15:53:27,082 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/851d0f1d522c4c3bb2ebe29791c5326b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/851d0f1d522c4c3bb2ebe29791c5326b 2024-11-07T15:53:27,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741874_1050 (size=12151) 2024-11-07T15:53:27,096 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/11af1e2c96224496a234161601883683 2024-11-07T15:53:27,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741875_1051 (size=12561) 2024-11-07T15:53:27,097 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 851d0f1d522c4c3bb2ebe29791c5326b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:27,097 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:27,097 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=12, startTime=1730994806996; duration=0sec 2024-11-07T15:53:27,097 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:27,097 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:27,097 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:27,102 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48878 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:27,102 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:27,102 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:27,102 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/fe7f50bd023c44bcb41ee0407ab5405a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=47.7 K 2024-11-07T15:53:27,104 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7f50bd023c44bcb41ee0407ab5405a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994804446 2024-11-07T15:53:27,105 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b7e31497a46347bba586ba9a1d51c822, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1730994805259 2024-11-07T15:53:27,105 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b610b2bd178849a9bed696aea3d1643c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1730994805357 2024-11-07T15:53:27,109 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b1b5c955d8a84568a52e273e8830dc55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805543 2024-11-07T15:53:27,113 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/d944a5fd90c44177a50d53d0dde1d82b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/d944a5fd90c44177a50d53d0dde1d82b 2024-11-07T15:53:27,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/aaacdfd392cc40bd915ea50365c4f0bd is 50, key is test_row_0/B:col10/1730994805686/Put/seqid=0 2024-11-07T15:53:27,125 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into d944a5fd90c44177a50d53d0dde1d82b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:27,125 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:27,125 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=12, startTime=1730994806996; duration=0sec 2024-11-07T15:53:27,125 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:27,125 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:27,152 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#40 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:27,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741876_1052 (size=12151) 2024-11-07T15:53:27,153 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/c3e50bb3b28b4aa1b246331965823529 is 50, key is test_row_0/C:col10/1730994805658/Put/seqid=0 2024-11-07T15:53:27,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/aaacdfd392cc40bd915ea50365c4f0bd 2024-11-07T15:53:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/30477f4a604f4dd8b1afda7ee5e531e4 is 50, key is test_row_0/C:col10/1730994805686/Put/seqid=0 2024-11-07T15:53:27,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741877_1053 (size=12561) 2024-11-07T15:53:27,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741878_1054 (size=12151) 2024-11-07T15:53:27,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/30477f4a604f4dd8b1afda7ee5e531e4 2024-11-07T15:53:27,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/11af1e2c96224496a234161601883683 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683 2024-11-07T15:53:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:27,233 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T15:53:27,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/aaacdfd392cc40bd915ea50365c4f0bd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd 2024-11-07T15:53:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T15:53:27,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/30477f4a604f4dd8b1afda7ee5e531e4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4 2024-11-07T15:53:27,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4, entries=150, sequenceid=208, filesize=11.9 K 2024-11-07T15:53:27,255 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for b2da43428956e53a96aa3524187a4a97 in 233ms, sequenceid=208, compaction requested=false 2024-11-07T15:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:27,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-07T15:53:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-07T15:53:27,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-07T15:53:27,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1660 sec 2024-11-07T15:53:27,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1770 sec 2024-11-07T15:53:27,613 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/c3e50bb3b28b4aa1b246331965823529 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/c3e50bb3b28b4aa1b246331965823529 2024-11-07T15:53:27,624 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into c3e50bb3b28b4aa1b246331965823529(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:27,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:27,624 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=12, startTime=1730994806998; duration=0sec 2024-11-07T15:53:27,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:27,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:27,873 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:27,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:53:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:27,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/82a2b4845487436495f90bb6fdd2d2e5 is 50, key is test_row_0/A:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:27,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994867925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:27,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741879_1055 (size=12151) 2024-11-07T15:53:27,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994867925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:27,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994867925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/82a2b4845487436495f90bb6fdd2d2e5 2024-11-07T15:53:27,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:27,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994867932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:27,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994867937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:27,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:27,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:27,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6f47a77277d241979b1ded88dd65955c is 50, key is test_row_0/B:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:27,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741880_1056 (size=12151) 2024-11-07T15:53:27,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6f47a77277d241979b1ded88dd65955c 2024-11-07T15:53:28,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/16a56521a55b43a6955a186960e9b885 is 50, key is test_row_0/C:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:28,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741881_1057 (size=12151) 2024-11-07T15:53:28,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/16a56521a55b43a6955a186960e9b885 2024-11-07T15:53:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/82a2b4845487436495f90bb6fdd2d2e5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5 2024-11-07T15:53:28,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T15:53:28,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6f47a77277d241979b1ded88dd65955c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c 2024-11-07T15:53:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994868044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994868044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994868044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994868044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994868044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T15:53:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/16a56521a55b43a6955a186960e9b885 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885 2024-11-07T15:53:28,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885, entries=150, sequenceid=222, filesize=11.9 K 2024-11-07T15:53:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b2da43428956e53a96aa3524187a4a97 in 189ms, sequenceid=222, compaction requested=true 2024-11-07T15:53:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:28,063 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:28,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:28,063 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:28,065 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:28,065 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:28,065 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:28,065 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:28,065 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,065 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,065 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/851d0f1d522c4c3bb2ebe29791c5326b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.0 K 2024-11-07T15:53:28,066 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/d944a5fd90c44177a50d53d0dde1d82b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.0 K 2024-11-07T15:53:28,066 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 851d0f1d522c4c3bb2ebe29791c5326b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805543 2024-11-07T15:53:28,067 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting aaacdfd392cc40bd915ea50365c4f0bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730994805683 2024-11-07T15:53:28,067 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting d944a5fd90c44177a50d53d0dde1d82b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805543 2024-11-07T15:53:28,067 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f47a77277d241979b1ded88dd65955c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:28,069 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11af1e2c96224496a234161601883683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730994805683 2024-11-07T15:53:28,070 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82a2b4845487436495f90bb6fdd2d2e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:28,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:28,094 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:28,094 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:28,095 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c1213bc3241144eb95d7563791b7e4a1 is 50, key is test_row_0/A:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:28,095 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2054f5d42ae7452a8b75eac39d7966f4 is 50, key is test_row_0/B:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:28,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741882_1058 (size=12663) 2024-11-07T15:53:28,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741883_1059 (size=12663) 2024-11-07T15:53:28,123 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2054f5d42ae7452a8b75eac39d7966f4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2054f5d42ae7452a8b75eac39d7966f4 2024-11-07T15:53:28,133 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 2054f5d42ae7452a8b75eac39d7966f4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:28,133 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:28,133 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=13, startTime=1730994808063; duration=0sec 2024-11-07T15:53:28,133 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:28,133 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:28,133 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:28,136 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:28,136 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:28,136 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,136 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/c3e50bb3b28b4aa1b246331965823529, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.0 K 2024-11-07T15:53:28,137 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e50bb3b28b4aa1b246331965823529, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1730994805543 2024-11-07T15:53:28,137 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 30477f4a604f4dd8b1afda7ee5e531e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1730994805683 2024-11-07T15:53:28,138 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a56521a55b43a6955a186960e9b885, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:28,164 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:28,165 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e256434bc55c48218b8fe278e74978d6 is 50, key is test_row_0/C:col10/1730994807868/Put/seqid=0 2024-11-07T15:53:28,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741884_1060 (size=12663) 2024-11-07T15:53:28,197 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e256434bc55c48218b8fe278e74978d6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e256434bc55c48218b8fe278e74978d6 2024-11-07T15:53:28,208 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into e256434bc55c48218b8fe278e74978d6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:28,208 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:28,208 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=13, startTime=1730994808074; duration=0sec 2024-11-07T15:53:28,208 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:28,208 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-07T15:53:28,233 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:28,234 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-07T15:53:28,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:28,237 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:28,239 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:28,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:28,256 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:28,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:53:28,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994868261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994868263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994868265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a6bdb15803ec4aefa2a480523aef67b2 is 50, key is test_row_0/A:col10/1730994808256/Put/seqid=0 2024-11-07T15:53:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994868270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994868270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741885_1061 (size=12151) 2024-11-07T15:53:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:28,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994868373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994868373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994868383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994868383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994868383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:28,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,519 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c1213bc3241144eb95d7563791b7e4a1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1213bc3241144eb95d7563791b7e4a1 2024-11-07T15:53:28,527 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into c1213bc3241144eb95d7563791b7e4a1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:28,527 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:28,527 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=13, startTime=1730994808063; duration=0sec 2024-11-07T15:53:28,528 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:28,528 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:28,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:28,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994868583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994868583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994868593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994868594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994868594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a6bdb15803ec4aefa2a480523aef67b2 2024-11-07T15:53:28,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/9fb53077d8ab4982b04f5025f6e73b0c is 50, key is test_row_0/B:col10/1730994808256/Put/seqid=0 2024-11-07T15:53:28,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741886_1062 (size=12151) 2024-11-07T15:53:28,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/9fb53077d8ab4982b04f5025f6e73b0c 2024-11-07T15:53:28,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e69ce9a0dbc1476e9efb037f220b92b5 is 50, key is test_row_0/C:col10/1730994808256/Put/seqid=0 2024-11-07T15:53:28,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741887_1063 (size=12151) 2024-11-07T15:53:28,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:28,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:28,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994868893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994868893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994868904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994868904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994868904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:28,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:29,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e69ce9a0dbc1476e9efb037f220b92b5 2024-11-07T15:53:29,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:29,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:29,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a6bdb15803ec4aefa2a480523aef67b2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2 2024-11-07T15:53:29,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:53:29,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/9fb53077d8ab4982b04f5025f6e73b0c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c 2024-11-07T15:53:29,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:53:29,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e69ce9a0dbc1476e9efb037f220b92b5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5 2024-11-07T15:53:29,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:53:29,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b2da43428956e53a96aa3524187a4a97 in 956ms, sequenceid=251, compaction requested=false 2024-11-07T15:53:29,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:29,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-07T15:53:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:29,326 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T15:53:29,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c1900ed5e052444f9470c0af040bf5c0 is 50, key is test_row_0/A:col10/1730994808258/Put/seqid=0 2024-11-07T15:53:29,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741888_1064 (size=9807) 2024-11-07T15:53:29,349 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c1900ed5e052444f9470c0af040bf5c0 2024-11-07T15:53:29,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c7b0a726c15e43aa9c95e1fd5f388da1 is 50, key is test_row_0/B:col10/1730994808258/Put/seqid=0 2024-11-07T15:53:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741889_1065 (size=9807) 2024-11-07T15:53:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:29,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:29,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994869439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994869445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994869446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994869447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994869448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994869553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994869563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994869564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994869564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994869564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994869763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994869773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,778 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c7b0a726c15e43aa9c95e1fd5f388da1 2024-11-07T15:53:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994869783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994869784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994869784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:29,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/39758bb6f1c346d1b55ec6705ca216bd is 50, key is test_row_0/C:col10/1730994808258/Put/seqid=0 2024-11-07T15:53:29,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741890_1066 (size=9807) 2024-11-07T15:53:30,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994870073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994870083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994870093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994870093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994870093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,196 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/39758bb6f1c346d1b55ec6705ca216bd 2024-11-07T15:53:30,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c1900ed5e052444f9470c0af040bf5c0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0 2024-11-07T15:53:30,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0, entries=100, sequenceid=261, filesize=9.6 K 2024-11-07T15:53:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/c7b0a726c15e43aa9c95e1fd5f388da1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1 2024-11-07T15:53:30,225 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1, entries=100, sequenceid=261, filesize=9.6 K 2024-11-07T15:53:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/39758bb6f1c346d1b55ec6705ca216bd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd 2024-11-07T15:53:30,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd, entries=100, sequenceid=261, filesize=9.6 K 2024-11-07T15:53:30,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for b2da43428956e53a96aa3524187a4a97 in 911ms, sequenceid=261, compaction requested=true 2024-11-07T15:53:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:30,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-07T15:53:30,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-07T15:53:30,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-07T15:53:30,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0010 sec 2024-11-07T15:53:30,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0120 sec 2024-11-07T15:53:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-07T15:53:30,384 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:30,385 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-07T15:53:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:53:30,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:30,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:30,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:53:30,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-07T15:53:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:30,542 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T15:53:30,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/87d0acaad9984023910fd29b4719c439 is 50, key is test_row_0/A:col10/1730994809446/Put/seqid=0 2024-11-07T15:53:30,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741891_1067 (size=12301) 2024-11-07T15:53:30,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/87d0acaad9984023910fd29b4719c439 2024-11-07T15:53:30,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3130f181a4cb480689d4fd8e2aaf042b is 50, key is test_row_0/B:col10/1730994809446/Put/seqid=0 2024-11-07T15:53:30,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741892_1068 (size=12301) 2024-11-07T15:53:30,582 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3130f181a4cb480689d4fd8e2aaf042b 2024-11-07T15:53:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:30,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:30,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1eb734f7316c4805bd315155e577ceb5 is 50, key is test_row_0/C:col10/1730994809446/Put/seqid=0 2024-11-07T15:53:30,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741893_1069 (size=12301) 2024-11-07T15:53:30,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994870601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994870602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994870613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994870613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994870613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:53:30,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994870714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994870714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994870923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994870923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:30,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,002 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1eb734f7316c4805bd315155e577ceb5 2024-11-07T15:53:31,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/87d0acaad9984023910fd29b4719c439 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439 2024-11-07T15:53:31,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:53:31,015 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:53:31,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3130f181a4cb480689d4fd8e2aaf042b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b 2024-11-07T15:53:31,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:53:31,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/1eb734f7316c4805bd315155e577ceb5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5 2024-11-07T15:53:31,029 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5, entries=150, sequenceid=289, filesize=12.0 K 2024-11-07T15:53:31,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b2da43428956e53a96aa3524187a4a97 in 488ms, sequenceid=289, compaction requested=true 2024-11-07T15:53:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-07T15:53:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-07T15:53:31,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-07T15:53:31,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-07T15:53:31,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 651 msec 2024-11-07T15:53:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:31,239 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:31,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:53:31,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/8aec1c06214e4b2db563f964ce7a5a6f is 50, key is test_row_0/A:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:31,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741894_1070 (size=14741) 2024-11-07T15:53:31,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994871304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994871306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994871413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994871413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-07T15:53:31,523 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:31,524 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:31,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-07T15:53:31,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:31,526 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:31,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:31,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994871623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994871624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994871624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,630 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994871633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994871633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,635 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,636 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/8aec1c06214e4b2db563f964ce7a5a6f 2024-11-07T15:53:31,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/e00f598ef588447f8b210a7f140c0718 is 50, key is test_row_0/B:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:31,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741895_1071 (size=12301) 2024-11-07T15:53:31,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/e00f598ef588447f8b210a7f140c0718 2024-11-07T15:53:31,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T15:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:31,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:31,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:31,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:31,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e840bb0b84394187b87fa1b947a58d91 is 50, key is test_row_0/C:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:31,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741896_1072 (size=12301) 2024-11-07T15:53:31,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e840bb0b84394187b87fa1b947a58d91 2024-11-07T15:53:31,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/8aec1c06214e4b2db563f964ce7a5a6f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f 2024-11-07T15:53:31,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f, entries=200, sequenceid=300, filesize=14.4 K 2024-11-07T15:53:31,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/e00f598ef588447f8b210a7f140c0718 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718 2024-11-07T15:53:31,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718, entries=150, sequenceid=300, filesize=12.0 K 2024-11-07T15:53:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e840bb0b84394187b87fa1b947a58d91 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91 2024-11-07T15:53:31,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91, entries=150, sequenceid=300, filesize=12.0 K 2024-11-07T15:53:31,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b2da43428956e53a96aa3524187a4a97 in 494ms, sequenceid=300, compaction requested=true 2024-11-07T15:53:31,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:31,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:31,733 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:53:31,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:31,734 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:53:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:31,736 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59223 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:53:31,736 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61663 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:53:31,736 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:31,736 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:31,736 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,736 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,736 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2054f5d42ae7452a8b75eac39d7966f4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=57.8 K 2024-11-07T15:53:31,736 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1213bc3241144eb95d7563791b7e4a1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=60.2 K 2024-11-07T15:53:31,737 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2054f5d42ae7452a8b75eac39d7966f4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:31,737 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb53077d8ab4982b04f5025f6e73b0c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994807906 2024-11-07T15:53:31,738 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b0a726c15e43aa9c95e1fd5f388da1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730994808258 2024-11-07T15:53:31,738 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3130f181a4cb480689d4fd8e2aaf042b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730994809436 2024-11-07T15:53:31,738 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1213bc3241144eb95d7563791b7e4a1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:31,739 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e00f598ef588447f8b210a7f140c0718, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:31,739 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6bdb15803ec4aefa2a480523aef67b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994807906 2024-11-07T15:53:31,740 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1900ed5e052444f9470c0af040bf5c0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730994808258 2024-11-07T15:53:31,741 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87d0acaad9984023910fd29b4719c439, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730994809436 2024-11-07T15:53:31,741 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aec1c06214e4b2db563f964ce7a5a6f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:31,759 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#60 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:31,760 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/d74c9110396e41e9b82da267db5d5a1e is 50, key is test_row_0/B:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:31,766 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:31,767 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5e29b167592a41be82fa02014da0b905 is 50, key is test_row_0/A:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:31,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741897_1073 (size=12983) 2024-11-07T15:53:31,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741898_1074 (size=12983) 2024-11-07T15:53:31,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-07T15:53:31,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:31,835 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:53:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/fa7acd52887946ea821eb3f8ec8b8fdf is 50, key is test_row_0/A:col10/1730994811295/Put/seqid=0 2024-11-07T15:53:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:31,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741899_1075 (size=12301) 2024-11-07T15:53:31,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/fa7acd52887946ea821eb3f8ec8b8fdf 2024-11-07T15:53:31,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/16c1009fff4b4273a97d475301c089cb is 50, key is test_row_0/B:col10/1730994811295/Put/seqid=0 2024-11-07T15:53:31,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741900_1076 (size=12301) 2024-11-07T15:53:31,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:31,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:31,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994871980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994871981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:31,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994872103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994872103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:32,189 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/d74c9110396e41e9b82da267db5d5a1e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d74c9110396e41e9b82da267db5d5a1e 2024-11-07T15:53:32,198 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/5e29b167592a41be82fa02014da0b905 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5e29b167592a41be82fa02014da0b905 2024-11-07T15:53:32,199 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into d74c9110396e41e9b82da267db5d5a1e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:32,199 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:32,199 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=11, startTime=1730994811733; duration=0sec 2024-11-07T15:53:32,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:32,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:32,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:53:32,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59223 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:53:32,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:32,202 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:32,202 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e256434bc55c48218b8fe278e74978d6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=57.8 K 2024-11-07T15:53:32,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e256434bc55c48218b8fe278e74978d6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1730994807858 2024-11-07T15:53:32,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e69ce9a0dbc1476e9efb037f220b92b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994807906 2024-11-07T15:53:32,204 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 39758bb6f1c346d1b55ec6705ca216bd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1730994808258 2024-11-07T15:53:32,205 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb734f7316c4805bd315155e577ceb5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1730994809436 2024-11-07T15:53:32,205 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e840bb0b84394187b87fa1b947a58d91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:32,206 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 5e29b167592a41be82fa02014da0b905(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:32,206 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:32,206 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=11, startTime=1730994811733; duration=0sec 2024-11-07T15:53:32,207 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:32,207 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:32,219 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:32,220 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e81e9cee417a4a9ba972079324c77a1f is 50, key is test_row_0/C:col10/1730994811237/Put/seqid=0 2024-11-07T15:53:32,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741901_1077 (size=12983) 2024-11-07T15:53:32,286 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/16c1009fff4b4273a97d475301c089cb 2024-11-07T15:53:32,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b204dd4610ad4a9bb8b19e7dd80dfc8d is 50, key is test_row_0/C:col10/1730994811295/Put/seqid=0 2024-11-07T15:53:32,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741902_1078 (size=12301) 2024-11-07T15:53:32,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994872313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994872313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994872633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994872634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:32,640 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/e81e9cee417a4a9ba972079324c77a1f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e81e9cee417a4a9ba972079324c77a1f 2024-11-07T15:53:32,648 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into e81e9cee417a4a9ba972079324c77a1f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:32,648 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:32,648 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=11, startTime=1730994811734; duration=0sec 2024-11-07T15:53:32,648 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:32,648 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:32,713 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b204dd4610ad4a9bb8b19e7dd80dfc8d 2024-11-07T15:53:32,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/fa7acd52887946ea821eb3f8ec8b8fdf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf 2024-11-07T15:53:32,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:53:32,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/16c1009fff4b4273a97d475301c089cb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb 2024-11-07T15:53:32,735 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:53:32,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/b204dd4610ad4a9bb8b19e7dd80dfc8d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d 2024-11-07T15:53:32,744 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d, entries=150, sequenceid=325, filesize=12.0 K 2024-11-07T15:53:32,745 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b2da43428956e53a96aa3524187a4a97 in 910ms, sequenceid=325, compaction requested=false 2024-11-07T15:53:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:32,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-07T15:53:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-07T15:53:32,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-07T15:53:32,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2200 sec 2024-11-07T15:53:32,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.2300 sec 2024-11-07T15:53:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:33,147 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:33,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:53:33,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e487242020a04426891b89533a31b6ef is 50, key is test_row_0/A:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741903_1079 (size=14741) 2024-11-07T15:53:33,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e487242020a04426891b89533a31b6ef 2024-11-07T15:53:33,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/53658cf287d54037bbf4546cb2b3a1d8 is 50, key is test_row_0/B:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741904_1080 (size=12301) 2024-11-07T15:53:33,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/53658cf287d54037bbf4546cb2b3a1d8 2024-11-07T15:53:33,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994873215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994873220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/25d52f3f54e04be08c8e3a8da25a671b is 50, key is test_row_0/C:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741905_1081 (size=12301) 2024-11-07T15:53:33,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/25d52f3f54e04be08c8e3a8da25a671b 2024-11-07T15:53:33,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e487242020a04426891b89533a31b6ef as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef 2024-11-07T15:53:33,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef, entries=200, sequenceid=341, filesize=14.4 K 2024-11-07T15:53:33,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/53658cf287d54037bbf4546cb2b3a1d8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8 2024-11-07T15:53:33,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8, entries=150, sequenceid=341, filesize=12.0 K 2024-11-07T15:53:33,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/25d52f3f54e04be08c8e3a8da25a671b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b 2024-11-07T15:53:33,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b, entries=150, sequenceid=341, filesize=12.0 K 2024-11-07T15:53:33,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b2da43428956e53a96aa3524187a4a97 in 122ms, sequenceid=341, compaction requested=true 2024-11-07T15:53:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:33,269 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:33,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:33,270 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:33,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:33,271 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:33,271 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:33,271 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:33,271 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:33,271 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:33,271 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,271 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,271 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5e29b167592a41be82fa02014da0b905, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=39.1 K 2024-11-07T15:53:33,271 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d74c9110396e41e9b82da267db5d5a1e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.7 K 2024-11-07T15:53:33,272 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e29b167592a41be82fa02014da0b905, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:33,272 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d74c9110396e41e9b82da267db5d5a1e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:33,272 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7acd52887946ea821eb3f8ec8b8fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730994811286 2024-11-07T15:53:33,272 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c1009fff4b4273a97d475301c089cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730994811286 2024-11-07T15:53:33,273 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e487242020a04426891b89533a31b6ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811976 2024-11-07T15:53:33,273 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 53658cf287d54037bbf4546cb2b3a1d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811978 2024-11-07T15:53:33,286 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:33,286 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/299b0d880f1247efba3b7c5da0a0f984 is 50, key is test_row_0/A:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,297 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:33,298 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/5517da70794641a9a04ff1dc209fcfef is 50, key is test_row_0/B:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741906_1082 (size=13085) 2024-11-07T15:53:33,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741907_1083 (size=13085) 2024-11-07T15:53:33,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:33,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:53:33,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/466d698f421b41efbeeaeaa2316e25ca is 50, key is test_row_0/A:col10/1730994813212/Put/seqid=0 2024-11-07T15:53:33,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741908_1084 (size=14741) 2024-11-07T15:53:33,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994873350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,356 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994873356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994873463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994873473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994873643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994873643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994873643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,645 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:33,645 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,647 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:33,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-07T15:53:33,674 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:33,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994873675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-07T15:53:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:33,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:33,681 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:33,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:33,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994873683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,710 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/299b0d880f1247efba3b7c5da0a0f984 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/299b0d880f1247efba3b7c5da0a0f984 2024-11-07T15:53:33,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/5517da70794641a9a04ff1dc209fcfef as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5517da70794641a9a04ff1dc209fcfef 2024-11-07T15:53:33,724 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 299b0d880f1247efba3b7c5da0a0f984(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:33,724 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:33,725 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=13, startTime=1730994813269; duration=0sec 2024-11-07T15:53:33,726 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:33,728 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:33,728 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:33,733 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:33,733 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:33,733 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,734 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e81e9cee417a4a9ba972079324c77a1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.7 K 2024-11-07T15:53:33,734 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 5517da70794641a9a04ff1dc209fcfef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:33,734 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81e9cee417a4a9ba972079324c77a1f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1730994810598 2024-11-07T15:53:33,734 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:33,735 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=13, startTime=1730994813270; duration=0sec 2024-11-07T15:53:33,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:33,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:33,736 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b204dd4610ad4a9bb8b19e7dd80dfc8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1730994811286 2024-11-07T15:53:33,738 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25d52f3f54e04be08c8e3a8da25a671b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811978 2024-11-07T15:53:33,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/466d698f421b41efbeeaeaa2316e25ca 2024-11-07T15:53:33,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3a0e8562512c4627ba0cdca2ad659c08 is 50, key is test_row_0/B:col10/1730994813212/Put/seqid=0 2024-11-07T15:53:33,769 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:33,770 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5b93a746e8ac427f9b666bf474fa94f8 is 50, key is test_row_0/C:col10/1730994811978/Put/seqid=0 2024-11-07T15:53:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741909_1085 (size=12301) 2024-11-07T15:53:33,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3a0e8562512c4627ba0cdca2ad659c08 2024-11-07T15:53:33,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741910_1086 (size=13085) 2024-11-07T15:53:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:33,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/3816c32557254ca8bca70590f38b1d2e is 50, key is test_row_0/C:col10/1730994813212/Put/seqid=0 2024-11-07T15:53:33,790 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5b93a746e8ac427f9b666bf474fa94f8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5b93a746e8ac427f9b666bf474fa94f8 2024-11-07T15:53:33,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741911_1087 (size=12301) 2024-11-07T15:53:33,811 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into 5b93a746e8ac427f9b666bf474fa94f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:33,811 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:33,811 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=13, startTime=1730994813271; duration=0sec 2024-11-07T15:53:33,811 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:33,811 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:33,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:53:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994873983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:33,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:33,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:53:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:33,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:34,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994874003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:53:34,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:34,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:34,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:34,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:34,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:34,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/3816c32557254ca8bca70590f38b1d2e 2024-11-07T15:53:34,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/466d698f421b41efbeeaeaa2316e25ca as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca 2024-11-07T15:53:34,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca, entries=200, sequenceid=365, filesize=14.4 K 2024-11-07T15:53:34,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3a0e8562512c4627ba0cdca2ad659c08 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08 2024-11-07T15:53:34,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:53:34,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/3816c32557254ca8bca70590f38b1d2e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e 2024-11-07T15:53:34,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-07T15:53:34,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b2da43428956e53a96aa3524187a4a97 in 903ms, sequenceid=365, compaction requested=false 2024-11-07T15:53:34,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:34,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-07T15:53:34,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:34,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:53:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:34,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/0573825d8615472fb0e55a4ef15bf52a is 50, key is test_row_0/A:col10/1730994813347/Put/seqid=0 2024-11-07T15:53:34,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741912_1088 (size=12301) 2024-11-07T15:53:34,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:34,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:34,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994874561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994874563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994874673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,675 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994874673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/0573825d8615472fb0e55a4ef15bf52a 2024-11-07T15:53:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/394698621be74801809bcfa0742ac7fd is 50, key is test_row_0/B:col10/1730994813347/Put/seqid=0 2024-11-07T15:53:34,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741913_1089 (size=12301) 2024-11-07T15:53:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:34,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994874883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994874883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,134 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/394698621be74801809bcfa0742ac7fd 2024-11-07T15:53:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c6ee2f8c1eb4a0980135d66c6feb173 is 50, key is test_row_0/C:col10/1730994813347/Put/seqid=0 2024-11-07T15:53:35,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741914_1090 (size=12301) 2024-11-07T15:53:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994875193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994875193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c6ee2f8c1eb4a0980135d66c6feb173 2024-11-07T15:53:35,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/0573825d8615472fb0e55a4ef15bf52a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a 2024-11-07T15:53:35,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T15:53:35,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/394698621be74801809bcfa0742ac7fd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd 2024-11-07T15:53:35,581 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T15:53:35,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/5c6ee2f8c1eb4a0980135d66c6feb173 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173 2024-11-07T15:53:35,588 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173, entries=150, sequenceid=380, filesize=12.0 K 2024-11-07T15:53:35,590 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b2da43428956e53a96aa3524187a4a97 in 1292ms, sequenceid=380, compaction requested=true 2024-11-07T15:53:35,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:35,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:35,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-07T15:53:35,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-07T15:53:35,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-07T15:53:35,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9110 sec 2024-11-07T15:53:35,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.9210 sec 2024-11-07T15:53:35,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:35,706 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:35,706 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:53:35,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/513ce9da583742f280f9a772ff7245e3 is 50, key is test_row_0/A:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:35,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741915_1091 (size=17181) 2024-11-07T15:53:35,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/513ce9da583742f280f9a772ff7245e3 2024-11-07T15:53:35,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2ac1bf3a58ba4058b8b26d83d7dba966 is 50, key is test_row_0/B:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994875732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994875733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741916_1092 (size=12301) 2024-11-07T15:53:35,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-07T15:53:35,833 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:35,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:35,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-07T15:53:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:35,836 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:35,837 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:35,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:35,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994875843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994875843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:35,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T15:53:35,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:35,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:35,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:35,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:35,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:36,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994876053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994876053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T15:53:36,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:36,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:36,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:36,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2ac1bf3a58ba4058b8b26d83d7dba966 2024-11-07T15:53:36,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:36,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/ba96ef846c934ee8a9ce9cffa7a95446 is 50, key is test_row_0/C:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:36,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741917_1093 (size=12301) 2024-11-07T15:53:36,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/ba96ef846c934ee8a9ce9cffa7a95446 2024-11-07T15:53:36,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/513ce9da583742f280f9a772ff7245e3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3 2024-11-07T15:53:36,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3, entries=250, sequenceid=406, filesize=16.8 K 2024-11-07T15:53:36,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2ac1bf3a58ba4058b8b26d83d7dba966 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966 2024-11-07T15:53:36,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T15:53:36,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/ba96ef846c934ee8a9ce9cffa7a95446 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446 2024-11-07T15:53:36,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446, entries=150, sequenceid=406, filesize=12.0 K 2024-11-07T15:53:36,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for b2da43428956e53a96aa3524187a4a97 in 492ms, sequenceid=406, compaction requested=true 2024-11-07T15:53:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:36,198 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:36,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:36,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:36,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:36,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:36,199 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:36,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:36,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:36,200 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:36,200 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:36,200 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,200 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/299b0d880f1247efba3b7c5da0a0f984, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=56.0 K 2024-11-07T15:53:36,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:36,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:36,201 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,201 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299b0d880f1247efba3b7c5da0a0f984, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811978 2024-11-07T15:53:36,201 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5517da70794641a9a04ff1dc209fcfef, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=48.8 K 2024-11-07T15:53:36,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5517da70794641a9a04ff1dc209fcfef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811978 2024-11-07T15:53:36,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466d698f421b41efbeeaeaa2316e25ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730994813198 2024-11-07T15:53:36,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0e8562512c4627ba0cdca2ad659c08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730994813198 2024-11-07T15:53:36,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0573825d8615472fb0e55a4ef15bf52a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730994813347 2024-11-07T15:53:36,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 394698621be74801809bcfa0742ac7fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730994813347 2024-11-07T15:53:36,203 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 513ce9da583742f280f9a772ff7245e3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:36,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac1bf3a58ba4058b8b26d83d7dba966, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:36,215 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:36,216 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2197e66384fa4dedbc8ca93854bf169c is 50, key is test_row_0/B:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:36,219 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:36,220 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/07c43b46e3764dbcbfb1d3fea2a0e180 is 50, key is test_row_0/A:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:36,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741919_1095 (size=13221) 2024-11-07T15:53:36,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741918_1094 (size=13221) 2024-11-07T15:53:36,238 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/07c43b46e3764dbcbfb1d3fea2a0e180 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/07c43b46e3764dbcbfb1d3fea2a0e180 2024-11-07T15:53:36,244 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 07c43b46e3764dbcbfb1d3fea2a0e180(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:36,244 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:36,244 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=12, startTime=1730994816198; duration=0sec 2024-11-07T15:53:36,244 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:36,245 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:36,245 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:36,246 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:36,246 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:36,246 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,247 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5b93a746e8ac427f9b666bf474fa94f8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=48.8 K 2024-11-07T15:53:36,248 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b93a746e8ac427f9b666bf474fa94f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1730994811978 2024-11-07T15:53:36,248 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3816c32557254ca8bca70590f38b1d2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1730994813198 2024-11-07T15:53:36,249 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c6ee2f8c1eb4a0980135d66c6feb173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1730994813347 2024-11-07T15:53:36,249 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba96ef846c934ee8a9ce9cffa7a95446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:36,259 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:36,259 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/6184fea0153145568ece3b97dcf4ec1e is 50, key is test_row_0/C:col10/1730994814540/Put/seqid=0 2024-11-07T15:53:36,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741920_1096 (size=13221) 2024-11-07T15:53:36,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-07T15:53:36,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:36,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:53:36,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/1a852209760a435bb66fbaeea173a33b is 50, key is test_row_0/A:col10/1730994815728/Put/seqid=0 2024-11-07T15:53:36,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741921_1097 (size=12301) 2024-11-07T15:53:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:36,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:36,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994876443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994876444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:36,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994876553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994876553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,638 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/2197e66384fa4dedbc8ca93854bf169c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2197e66384fa4dedbc8ca93854bf169c 2024-11-07T15:53:36,644 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 2197e66384fa4dedbc8ca93854bf169c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:36,645 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:36,645 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=12, startTime=1730994816199; duration=0sec 2024-11-07T15:53:36,645 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:36,645 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:36,671 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/6184fea0153145568ece3b97dcf4ec1e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6184fea0153145568ece3b97dcf4ec1e 2024-11-07T15:53:36,677 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into 6184fea0153145568ece3b97dcf4ec1e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:36,677 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:36,677 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=12, startTime=1730994816199; duration=0sec 2024-11-07T15:53:36,677 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:36,677 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:36,709 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/1a852209760a435bb66fbaeea173a33b 2024-11-07T15:53:36,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/4a71df732b104bd9ba2684ee93503a3a is 50, key is test_row_0/B:col10/1730994815728/Put/seqid=0 2024-11-07T15:53:36,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741922_1098 (size=12301) 2024-11-07T15:53:36,728 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/4a71df732b104bd9ba2684ee93503a3a 2024-11-07T15:53:36,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/af0a2b4f98544823a3e356fa23ff50a0 is 50, key is test_row_0/C:col10/1730994815728/Put/seqid=0 2024-11-07T15:53:36,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741923_1099 (size=12301) 2024-11-07T15:53:36,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994876763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994876763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:36,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994877073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994877073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,146 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/af0a2b4f98544823a3e356fa23ff50a0 2024-11-07T15:53:37,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/1a852209760a435bb66fbaeea173a33b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b 2024-11-07T15:53:37,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T15:53:37,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/4a71df732b104bd9ba2684ee93503a3a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a 2024-11-07T15:53:37,166 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T15:53:37,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/af0a2b4f98544823a3e356fa23ff50a0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0 2024-11-07T15:53:37,173 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0, entries=150, sequenceid=418, filesize=12.0 K 2024-11-07T15:53:37,175 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b2da43428956e53a96aa3524187a4a97 in 877ms, sequenceid=418, compaction requested=false 2024-11-07T15:53:37,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:37,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:37,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-07T15:53:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-07T15:53:37,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-07T15:53:37,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-11-07T15:53:37,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3460 sec 2024-11-07T15:53:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:37,587 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:37,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:53:37,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6ebd449b25fb450b8b27d1e4cddfc63c is 50, key is test_row_0/A:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994877600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994877602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741924_1100 (size=14741) 2024-11-07T15:53:37,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6ebd449b25fb450b8b27d1e4cddfc63c 2024-11-07T15:53:37,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6e4f8235bf914b07a6cccf9a369c68bb is 50, key is test_row_0/B:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:37,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741925_1101 (size=12301) 2024-11-07T15:53:37,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42284 deadline: 1730994877663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,666 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42280 deadline: 1730994877683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,685 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42310 deadline: 1730994877692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,698 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in b2da43428956e53a96aa3524187a4a97 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994877713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994877713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994877923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994877923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:37,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-07T15:53:37,984 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:37,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-07T15:53:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:37,986 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:37,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:37,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:38,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6e4f8235bf914b07a6cccf9a369c68bb 2024-11-07T15:53:38,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/f4777faac9d64f088802b1ead8943fd1 is 50, key is test_row_0/C:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:38,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741926_1102 (size=12301) 2024-11-07T15:53:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:38,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:53:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994878233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994878233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,240 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:53:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:38,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/f4777faac9d64f088802b1ead8943fd1 2024-11-07T15:53:38,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:53:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:38,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/6ebd449b25fb450b8b27d1e4cddfc63c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c 2024-11-07T15:53:38,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c, entries=200, sequenceid=447, filesize=14.4 K 2024-11-07T15:53:38,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/6e4f8235bf914b07a6cccf9a369c68bb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb 2024-11-07T15:53:38,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb, entries=150, sequenceid=447, filesize=12.0 K 2024-11-07T15:53:38,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/f4777faac9d64f088802b1ead8943fd1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1 2024-11-07T15:53:38,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1, entries=150, sequenceid=447, filesize=12.0 K 2024-11-07T15:53:38,471 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b2da43428956e53a96aa3524187a4a97 in 884ms, sequenceid=447, compaction requested=true 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:38,472 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:38,472 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:38,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:38,473 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:38,473 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:38,473 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:38,473 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:38,473 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,473 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,474 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/07c43b46e3764dbcbfb1d3fea2a0e180, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=39.3 K 2024-11-07T15:53:38,474 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2197e66384fa4dedbc8ca93854bf169c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.9 K 2024-11-07T15:53:38,474 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2197e66384fa4dedbc8ca93854bf169c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:38,474 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07c43b46e3764dbcbfb1d3fea2a0e180, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:38,474 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a71df732b104bd9ba2684ee93503a3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730994815720 2024-11-07T15:53:38,475 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a852209760a435bb66fbaeea173a33b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730994815720 2024-11-07T15:53:38,475 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4f8235bf914b07a6cccf9a369c68bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:38,475 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ebd449b25fb450b8b27d1e4cddfc63c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:38,486 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:38,487 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c42c5d70e1b24e9f82d3d066767e35ee is 50, key is test_row_0/A:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:38,488 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:38,489 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/466e366f65cc42928733f0bbe91cf20c is 50, key is test_row_0/B:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:38,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741927_1103 (size=13323) 2024-11-07T15:53:38,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741928_1104 (size=13323) 2024-11-07T15:53:38,504 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/466e366f65cc42928733f0bbe91cf20c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/466e366f65cc42928733f0bbe91cf20c 2024-11-07T15:53:38,510 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/c42c5d70e1b24e9f82d3d066767e35ee as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c42c5d70e1b24e9f82d3d066767e35ee 2024-11-07T15:53:38,512 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 466e366f65cc42928733f0bbe91cf20c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:38,512 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:38,512 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=13, startTime=1730994818472; duration=0sec 2024-11-07T15:53:38,512 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:38,512 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:38,512 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:38,514 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:38,514 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:38,514 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,514 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6184fea0153145568ece3b97dcf4ec1e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=36.9 K 2024-11-07T15:53:38,515 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6184fea0153145568ece3b97dcf4ec1e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1730994814540 2024-11-07T15:53:38,515 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting af0a2b4f98544823a3e356fa23ff50a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1730994815720 2024-11-07T15:53:38,516 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting f4777faac9d64f088802b1ead8943fd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:38,520 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into c42c5d70e1b24e9f82d3d066767e35ee(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:38,520 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:38,520 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=13, startTime=1730994818472; duration=0sec 2024-11-07T15:53:38,520 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:38,520 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:38,528 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:38,528 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/93570c3e7eb34ab2a1617411b5251e1f is 50, key is test_row_0/C:col10/1730994816430/Put/seqid=0 2024-11-07T15:53:38,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741929_1105 (size=13323) 2024-11-07T15:53:38,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-07T15:53:38,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:38,601 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T15:53:38,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/24353cb7c7e5439d8a2b5a32aeb33de7 is 50, key is test_row_0/A:col10/1730994817597/Put/seqid=0 2024-11-07T15:53:38,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:38,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741930_1106 (size=12301) 2024-11-07T15:53:38,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:38,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994878799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994878800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994878913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994878913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:38,917 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:38,948 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/93570c3e7eb34ab2a1617411b5251e1f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/93570c3e7eb34ab2a1617411b5251e1f 2024-11-07T15:53:38,954 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into 93570c3e7eb34ab2a1617411b5251e1f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:38,954 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:38,954 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=13, startTime=1730994818472; duration=0sec 2024-11-07T15:53:38,954 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:38,954 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:39,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/24353cb7c7e5439d8a2b5a32aeb33de7 2024-11-07T15:53:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/f3d9701b8bb64127a9531754d2d4ad36 is 50, key is test_row_0/B:col10/1730994817597/Put/seqid=0 2024-11-07T15:53:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741931_1107 (size=12301) 2024-11-07T15:53:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:39,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994879123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994879123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:39,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994879433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994879433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:39,438 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/f3d9701b8bb64127a9531754d2d4ad36 2024-11-07T15:53:39,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/7f3378788c08429da8f28c0bbf18457b is 50, key is test_row_0/C:col10/1730994817597/Put/seqid=0 2024-11-07T15:53:39,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741932_1108 (size=12301) 2024-11-07T15:53:39,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/7f3378788c08429da8f28c0bbf18457b 2024-11-07T15:53:39,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/24353cb7c7e5439d8a2b5a32aeb33de7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7 2024-11-07T15:53:39,864 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T15:53:39,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/f3d9701b8bb64127a9531754d2d4ad36 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36 2024-11-07T15:53:39,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T15:53:39,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/7f3378788c08429da8f28c0bbf18457b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b 2024-11-07T15:53:39,877 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b, entries=150, sequenceid=458, filesize=12.0 K 2024-11-07T15:53:39,878 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b2da43428956e53a96aa3524187a4a97 in 1277ms, sequenceid=458, compaction requested=false 2024-11-07T15:53:39,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:39,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-07T15:53:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-07T15:53:39,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-07T15:53:39,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8930 sec 2024-11-07T15:53:39,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.8990 sec 2024-11-07T15:53:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:39,944 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:39,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-07T15:53:39,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/98b98ffb29774c58b2a801ad0121225b is 50, key is test_row_0/A:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:39,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994879955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741933_1109 (size=14741) 2024-11-07T15:53:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:39,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994879960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:39,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994880063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994880073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-07T15:53:40,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:40,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-07T15:53:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:40,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:40,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:40,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:40,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:40,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994880273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994880283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T15:53:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:40,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/98b98ffb29774c58b2a801ad0121225b 2024-11-07T15:53:40,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/91646b0c5a434859b045525490790a51 is 50, key is test_row_0/B:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:40,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741934_1110 (size=12301) 2024-11-07T15:53:40,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/91646b0c5a434859b045525490790a51 2024-11-07T15:53:40,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/6e3b3e5c76074533823d285c9df7540d is 50, key is test_row_0/C:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:40,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741935_1111 (size=12301) 2024-11-07T15:53:40,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T15:53:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994880583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994880593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:40,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:40,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T15:53:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T15:53:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:40,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:40,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:40,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/6e3b3e5c76074533823d285c9df7540d 2024-11-07T15:53:40,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/98b98ffb29774c58b2a801ad0121225b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b 2024-11-07T15:53:40,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b, entries=200, sequenceid=488, filesize=14.4 K 2024-11-07T15:53:40,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/91646b0c5a434859b045525490790a51 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51 2024-11-07T15:53:40,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51, entries=150, sequenceid=488, filesize=12.0 K 2024-11-07T15:53:40,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/6e3b3e5c76074533823d285c9df7540d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d 2024-11-07T15:53:40,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d, entries=150, sequenceid=488, filesize=12.0 K 2024-11-07T15:53:40,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for b2da43428956e53a96aa3524187a4a97 in 907ms, sequenceid=488, compaction requested=true 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:40,852 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:40,852 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:40,853 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:40,853 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:40,853 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:40,853 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:40,853 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,853 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,854 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c42c5d70e1b24e9f82d3d066767e35ee, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=39.4 K 2024-11-07T15:53:40,854 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/466e366f65cc42928733f0bbe91cf20c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=37.0 K 2024-11-07T15:53:40,854 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42c5d70e1b24e9f82d3d066767e35ee, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:40,855 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 466e366f65cc42928733f0bbe91cf20c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:40,855 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24353cb7c7e5439d8a2b5a32aeb33de7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730994817597 2024-11-07T15:53:40,855 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting f3d9701b8bb64127a9531754d2d4ad36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730994817597 2024-11-07T15:53:40,855 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98b98ffb29774c58b2a801ad0121225b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:40,855 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 91646b0c5a434859b045525490790a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:40,870 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:40,871 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:40,872 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/688435e9439b43a8b3f59fbe91d016bf is 50, key is test_row_0/B:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:40,872 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a83feef568b74c0cb858eeafd8fa7b68 is 50, key is test_row_0/A:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741936_1112 (size=13425) 2024-11-07T15:53:40,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741937_1113 (size=13425) 2024-11-07T15:53:40,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-07T15:53:40,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:40,906 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T15:53:40,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/91b3adced20c48f890dc9233f83b45e7 is 50, key is test_row_0/A:col10/1730994819949/Put/seqid=0 2024-11-07T15:53:40,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741938_1114 (size=12301) 2024-11-07T15:53:40,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/91b3adced20c48f890dc9233f83b45e7 2024-11-07T15:53:40,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3cc74edac35a4fd5b2aa7e656366756d is 50, key is test_row_0/B:col10/1730994819949/Put/seqid=0 2024-11-07T15:53:40,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741939_1115 (size=12301) 2024-11-07T15:53:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:41,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. as already flushing 2024-11-07T15:53:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994881164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994881166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:41,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994881273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994881273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,295 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/688435e9439b43a8b3f59fbe91d016bf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/688435e9439b43a8b3f59fbe91d016bf 2024-11-07T15:53:41,295 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/a83feef568b74c0cb858eeafd8fa7b68 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a83feef568b74c0cb858eeafd8fa7b68 2024-11-07T15:53:41,305 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 688435e9439b43a8b3f59fbe91d016bf(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:41,305 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into a83feef568b74c0cb858eeafd8fa7b68(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:41,305 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=13, startTime=1730994820852; duration=0sec 2024-11-07T15:53:41,305 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=13, startTime=1730994820852; duration=0sec 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:41,305 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:41,306 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:41,306 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:41,306 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:41,307 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/93570c3e7eb34ab2a1617411b5251e1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=37.0 K 2024-11-07T15:53:41,307 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 93570c3e7eb34ab2a1617411b5251e1f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=447, earliestPutTs=1730994816430 2024-11-07T15:53:41,307 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3378788c08429da8f28c0bbf18457b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1730994817597 2024-11-07T15:53:41,308 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e3b3e5c76074533823d285c9df7540d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:41,317 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:41,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/d772f2192d854394b2a3dbabd8a7a6b8 is 50, key is test_row_0/C:col10/1730994818799/Put/seqid=0 2024-11-07T15:53:41,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741940_1116 (size=13425) 2024-11-07T15:53:41,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3cc74edac35a4fd5b2aa7e656366756d 2024-11-07T15:53:41,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/26126382d3994ff1b9969b4bcaffb4d8 is 50, key is test_row_0/C:col10/1730994819949/Put/seqid=0 2024-11-07T15:53:41,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741941_1117 (size=12301) 2024-11-07T15:53:41,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42342 deadline: 1730994881493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42294 deadline: 1730994881493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:53:41,677 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-07T15:53:41,677 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,678 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-07T15:53:41,678 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,679 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-07T15:53:41,679 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,681 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-07T15:53:41,681 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,682 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,684 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,730 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/d772f2192d854394b2a3dbabd8a7a6b8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d772f2192d854394b2a3dbabd8a7a6b8 2024-11-07T15:53:41,735 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into d772f2192d854394b2a3dbabd8a7a6b8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:41,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:41,735 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=13, startTime=1730994820852; duration=0sec 2024-11-07T15:53:41,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:41,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:41,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/26126382d3994ff1b9969b4bcaffb4d8 2024-11-07T15:53:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/91b3adced20c48f890dc9233f83b45e7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7 2024-11-07T15:53:41,786 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7, entries=150, sequenceid=495, filesize=12.0 K 2024-11-07T15:53:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3cc74edac35a4fd5b2aa7e656366756d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d 2024-11-07T15:53:41,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d, entries=150, sequenceid=495, filesize=12.0 K 2024-11-07T15:53:41,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/26126382d3994ff1b9969b4bcaffb4d8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8 2024-11-07T15:53:41,796 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8, entries=150, sequenceid=495, filesize=12.0 K 2024-11-07T15:53:41,797 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for b2da43428956e53a96aa3524187a4a97 in 890ms, sequenceid=495, compaction requested=false 2024-11-07T15:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-07T15:53:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-07T15:53:41,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-07T15:53:41,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6600 sec 2024-11-07T15:53:41,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.6660 sec 2024-11-07T15:53:41,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:41,804 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:41,804 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-07T15:53:41,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-07T15:53:41,804 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,804 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,804 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-07T15:53:41,804 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,804 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,804 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:41,804 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,804 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:41,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:41,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/249b96fb04a44f0584860b50a77d2db0 is 50, key is test_row_0/A:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:41,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741942_1118 (size=12301) 2024-11-07T15:53:42,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/249b96fb04a44f0584860b50a77d2db0 2024-11-07T15:53:42,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/587b3e01e2a04aa49828199c235e8e17 is 50, key is test_row_0/B:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:42,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741943_1119 (size=12301) 2024-11-07T15:53:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-07T15:53:42,293 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:42,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/587b3e01e2a04aa49828199c235e8e17 2024-11-07T15:53:42,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/64118f1a273d45dfb10404ca0cd74657 is 50, key is test_row_0/C:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:42,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741944_1120 (size=12301) 2024-11-07T15:53:43,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/64118f1a273d45dfb10404ca0cd74657 2024-11-07T15:53:43,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/249b96fb04a44f0584860b50a77d2db0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0 2024-11-07T15:53:43,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0, entries=150, sequenceid=529, filesize=12.0 K 2024-11-07T15:53:43,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/587b3e01e2a04aa49828199c235e8e17 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17 2024-11-07T15:53:43,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17, entries=150, sequenceid=529, filesize=12.0 K 2024-11-07T15:53:43,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/64118f1a273d45dfb10404ca0cd74657 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657 2024-11-07T15:53:43,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657, entries=150, sequenceid=529, filesize=12.0 K 2024-11-07T15:53:43,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=0 B/0 for b2da43428956e53a96aa3524187a4a97 in 1257ms, sequenceid=529, compaction requested=true 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:43,061 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b2da43428956e53a96aa3524187a4a97:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:43,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:43,061 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:43,062 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:43,062 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:43,062 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/B is initiating minor compaction (all files) 2024-11-07T15:53:43,062 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/A is initiating minor compaction (all files) 2024-11-07T15:53:43,062 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/B in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:43,062 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/A in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:43,062 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/688435e9439b43a8b3f59fbe91d016bf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=37.1 K 2024-11-07T15:53:43,063 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a83feef568b74c0cb858eeafd8fa7b68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=37.1 K 2024-11-07T15:53:43,063 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 688435e9439b43a8b3f59fbe91d016bf, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:43,063 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a83feef568b74c0cb858eeafd8fa7b68, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:43,063 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cc74edac35a4fd5b2aa7e656366756d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1730994819949 2024-11-07T15:53:43,063 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91b3adced20c48f890dc9233f83b45e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1730994819949 2024-11-07T15:53:43,064 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 587b3e01e2a04aa49828199c235e8e17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1730994821163 2024-11-07T15:53:43,064 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249b96fb04a44f0584860b50a77d2db0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1730994821163 2024-11-07T15:53:43,072 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#B#compaction#108 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:43,072 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#A#compaction#109 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:43,073 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/0033c8b3ef7543f3a043e9438f8587e7 is 50, key is test_row_0/A:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:43,073 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3cc06552c0b64e88bdfce2ee42bf113b is 50, key is test_row_0/B:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:43,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741945_1121 (size=13527) 2024-11-07T15:53:43,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741946_1122 (size=13527) 2024-11-07T15:53:43,486 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/0033c8b3ef7543f3a043e9438f8587e7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0033c8b3ef7543f3a043e9438f8587e7 2024-11-07T15:53:43,488 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/3cc06552c0b64e88bdfce2ee42bf113b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc06552c0b64e88bdfce2ee42bf113b 2024-11-07T15:53:43,492 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/A of b2da43428956e53a96aa3524187a4a97 into 0033c8b3ef7543f3a043e9438f8587e7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:43,492 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:43,492 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/A, priority=13, startTime=1730994823061; duration=0sec 2024-11-07T15:53:43,492 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:43,492 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:A 2024-11-07T15:53:43,492 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:43,493 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:43,493 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): b2da43428956e53a96aa3524187a4a97/C is initiating minor compaction (all files) 2024-11-07T15:53:43,493 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b2da43428956e53a96aa3524187a4a97/C in TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:43,493 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d772f2192d854394b2a3dbabd8a7a6b8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp, totalSize=37.1 K 2024-11-07T15:53:43,493 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/B of b2da43428956e53a96aa3524187a4a97 into 3cc06552c0b64e88bdfce2ee42bf113b(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:43,493 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:43,493 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/B, priority=13, startTime=1730994823061; duration=0sec 2024-11-07T15:53:43,494 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:43,494 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting d772f2192d854394b2a3dbabd8a7a6b8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1730994818797 2024-11-07T15:53:43,494 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:B 2024-11-07T15:53:43,494 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26126382d3994ff1b9969b4bcaffb4d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1730994819949 2024-11-07T15:53:43,494 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64118f1a273d45dfb10404ca0cd74657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1730994821163 2024-11-07T15:53:43,502 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b2da43428956e53a96aa3524187a4a97#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:43,503 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/4512e5bf512b415387de4fb7f940bc42 is 50, key is test_row_0/C:col10/1730994821803/Put/seqid=0 2024-11-07T15:53:43,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741947_1123 (size=13527) 2024-11-07T15:53:43,913 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/4512e5bf512b415387de4fb7f940bc42 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4512e5bf512b415387de4fb7f940bc42 2024-11-07T15:53:43,918 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b2da43428956e53a96aa3524187a4a97/C of b2da43428956e53a96aa3524187a4a97 into 4512e5bf512b415387de4fb7f940bc42(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:43,918 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b2da43428956e53a96aa3524187a4a97: 2024-11-07T15:53:43,918 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., storeName=b2da43428956e53a96aa3524187a4a97/C, priority=13, startTime=1730994823061; duration=0sec 2024-11-07T15:53:43,918 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:43,918 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b2da43428956e53a96aa3524187a4a97:C 2024-11-07T15:53:44,925 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:53:47,694 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-07T15:53:47,694 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:47,694 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,695 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,695 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:47,745 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-07T15:53:47,745 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:47,745 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,745 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,745 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:47,784 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-07T15:53:47,784 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:53:47,784 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,785 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 139 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3462 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3231 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1504 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4509 rows 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1519 2024-11-07T15:53:47,785 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4555 rows 2024-11-07T15:53:47,785 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:53:47,786 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:53:47,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:47,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:53:47,790 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:53:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:47,798 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994827797"}]},"ts":"1730994827797"} 2024-11-07T15:53:47,800 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:53:47,800 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:53:47,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:53:47,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, UNASSIGN}] 2024-11-07T15:53:47,806 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, UNASSIGN 2024-11-07T15:53:47,808 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b2da43428956e53a96aa3524187a4a97, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:47,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, UNASSIGN because future has completed 2024-11-07T15:53:47,810 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:53:47,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:47,967 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:47,967 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:53:47,968 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing b2da43428956e53a96aa3524187a4a97, disabling compactions & flushes 2024-11-07T15:53:47,968 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:47,968 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:47,968 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. after waiting 0 ms 2024-11-07T15:53:47,968 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:47,968 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing b2da43428956e53a96aa3524187a4a97 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:53:47,972 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e9958272fc724bba84d81cc28fbedd93 is 50, key is test_row_1/A:col10/1730994827744/Put/seqid=0 2024-11-07T15:53:47,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741948_1124 (size=9857) 2024-11-07T15:53:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:48,377 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e9958272fc724bba84d81cc28fbedd93 2024-11-07T15:53:48,386 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/a26b41b3e55143e9bf9e98e1c1b149f1 is 50, key is test_row_1/B:col10/1730994827744/Put/seqid=0 2024-11-07T15:53:48,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741949_1125 (size=9857) 2024-11-07T15:53:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:48,790 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/a26b41b3e55143e9bf9e98e1c1b149f1 2024-11-07T15:53:48,798 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/d5488d104eae4308896f7fea09379060 is 50, key is test_row_1/C:col10/1730994827744/Put/seqid=0 2024-11-07T15:53:48,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741950_1126 (size=9857) 2024-11-07T15:53:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:49,203 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/d5488d104eae4308896f7fea09379060 2024-11-07T15:53:49,208 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/A/e9958272fc724bba84d81cc28fbedd93 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e9958272fc724bba84d81cc28fbedd93 2024-11-07T15:53:49,212 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e9958272fc724bba84d81cc28fbedd93, entries=100, sequenceid=538, filesize=9.6 K 2024-11-07T15:53:49,213 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/B/a26b41b3e55143e9bf9e98e1c1b149f1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/a26b41b3e55143e9bf9e98e1c1b149f1 2024-11-07T15:53:49,218 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/a26b41b3e55143e9bf9e98e1c1b149f1, entries=100, sequenceid=538, filesize=9.6 K 2024-11-07T15:53:49,219 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/.tmp/C/d5488d104eae4308896f7fea09379060 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d5488d104eae4308896f7fea09379060 2024-11-07T15:53:49,223 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d5488d104eae4308896f7fea09379060, entries=100, sequenceid=538, filesize=9.6 K 2024-11-07T15:53:49,224 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b2da43428956e53a96aa3524187a4a97 in 1256ms, sequenceid=538, compaction requested=false 2024-11-07T15:53:49,225 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7cf2aefe184b4f7ca693bfe0548847e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/364ba24a084144ff836a38b37026235d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/d944a5fd90c44177a50d53d0dde1d82b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1213bc3241144eb95d7563791b7e4a1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5e29b167592a41be82fa02014da0b905, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/299b0d880f1247efba3b7c5da0a0f984, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/07c43b46e3764dbcbfb1d3fea2a0e180, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c42c5d70e1b24e9f82d3d066767e35ee, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a83feef568b74c0cb858eeafd8fa7b68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0] to archive 2024-11-07T15:53:49,228 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:53:49,234 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5b512bf9b0de415f987179d7873a9cda 2024-11-07T15:53:49,235 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/02b0abc544394e81842ca1cbb87d292d 2024-11-07T15:53:49,237 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7de29799a0b341069682da18fbe09268 2024-11-07T15:53:49,238 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7cf2aefe184b4f7ca693bfe0548847e8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/7cf2aefe184b4f7ca693bfe0548847e8 2024-11-07T15:53:49,240 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5f449e9dfa7c4a0cbf94e591a9191fe7 2024-11-07T15:53:49,241 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a55f244d93e041b289b82cdd60ae84c7 2024-11-07T15:53:49,243 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/cc75c8b831644738998a4563c7454857 2024-11-07T15:53:49,244 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6e3af31186744ddb99b841c359fd4066 2024-11-07T15:53:49,245 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/364ba24a084144ff836a38b37026235d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/364ba24a084144ff836a38b37026235d 2024-11-07T15:53:49,246 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/484d98a8d4144b5cb684dad48c2943fe 2024-11-07T15:53:49,247 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/068f00e0bf7b4b74895360f60d3a7750 2024-11-07T15:53:49,248 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e8d26aef6fb84c86881cb511df4d827f 2024-11-07T15:53:49,250 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/d944a5fd90c44177a50d53d0dde1d82b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/d944a5fd90c44177a50d53d0dde1d82b 2024-11-07T15:53:49,251 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/11af1e2c96224496a234161601883683 2024-11-07T15:53:49,253 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1213bc3241144eb95d7563791b7e4a1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1213bc3241144eb95d7563791b7e4a1 2024-11-07T15:53:49,254 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/82a2b4845487436495f90bb6fdd2d2e5 2024-11-07T15:53:49,255 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a6bdb15803ec4aefa2a480523aef67b2 2024-11-07T15:53:49,257 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c1900ed5e052444f9470c0af040bf5c0 2024-11-07T15:53:49,258 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/87d0acaad9984023910fd29b4719c439 2024-11-07T15:53:49,260 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/8aec1c06214e4b2db563f964ce7a5a6f 2024-11-07T15:53:49,261 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5e29b167592a41be82fa02014da0b905 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/5e29b167592a41be82fa02014da0b905 2024-11-07T15:53:49,262 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/fa7acd52887946ea821eb3f8ec8b8fdf 2024-11-07T15:53:49,264 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e487242020a04426891b89533a31b6ef 2024-11-07T15:53:49,265 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/299b0d880f1247efba3b7c5da0a0f984 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/299b0d880f1247efba3b7c5da0a0f984 2024-11-07T15:53:49,267 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/466d698f421b41efbeeaeaa2316e25ca 2024-11-07T15:53:49,268 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0573825d8615472fb0e55a4ef15bf52a 2024-11-07T15:53:49,270 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/513ce9da583742f280f9a772ff7245e3 2024-11-07T15:53:49,271 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/07c43b46e3764dbcbfb1d3fea2a0e180 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/07c43b46e3764dbcbfb1d3fea2a0e180 2024-11-07T15:53:49,273 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/1a852209760a435bb66fbaeea173a33b 2024-11-07T15:53:49,274 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/6ebd449b25fb450b8b27d1e4cddfc63c 2024-11-07T15:53:49,275 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c42c5d70e1b24e9f82d3d066767e35ee to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/c42c5d70e1b24e9f82d3d066767e35ee 2024-11-07T15:53:49,277 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/24353cb7c7e5439d8a2b5a32aeb33de7 2024-11-07T15:53:49,278 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/98b98ffb29774c58b2a801ad0121225b 2024-11-07T15:53:49,279 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a83feef568b74c0cb858eeafd8fa7b68 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/a83feef568b74c0cb858eeafd8fa7b68 2024-11-07T15:53:49,280 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/91b3adced20c48f890dc9233f83b45e7 2024-11-07T15:53:49,282 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/249b96fb04a44f0584860b50a77d2db0 2024-11-07T15:53:49,300 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c934c91ab5544dd5a61764799a8aa4c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4b18b4874d904bfe8728cfab169da2b2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/851d0f1d522c4c3bb2ebe29791c5326b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2054f5d42ae7452a8b75eac39d7966f4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d74c9110396e41e9b82da267db5d5a1e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5517da70794641a9a04ff1dc209fcfef, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2197e66384fa4dedbc8ca93854bf169c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/466e366f65cc42928733f0bbe91cf20c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/688435e9439b43a8b3f59fbe91d016bf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17] to archive 2024-11-07T15:53:49,302 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:53:49,304 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2d2313e531df499888ce1574c5cb8e5f 2024-11-07T15:53:49,305 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5f37346914af4059b12b22168a1145ea 2024-11-07T15:53:49,307 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3d6c31a762ac4c5c961d6fd9deaea4f5 2024-11-07T15:53:49,308 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c934c91ab5544dd5a61764799a8aa4c7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c934c91ab5544dd5a61764799a8aa4c7 2024-11-07T15:53:49,309 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d96993b71c6c474285fea209d3e9464d 2024-11-07T15:53:49,311 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/ffc8feebcbde4ecdbad23e1c5250fa15 2024-11-07T15:53:49,312 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/10e9465fed914b75ad28091507bd3935 2024-11-07T15:53:49,313 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4b18b4874d904bfe8728cfab169da2b2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4b18b4874d904bfe8728cfab169da2b2 2024-11-07T15:53:49,314 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/b5aa2c439569476bb3e2323005e3f9ee 2024-11-07T15:53:49,315 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/86ceab5dd2a740ada91414fd30ff67da 2024-11-07T15:53:49,316 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/28517390cec144a59a3ee50e83edfaf5 2024-11-07T15:53:49,317 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/851d0f1d522c4c3bb2ebe29791c5326b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/851d0f1d522c4c3bb2ebe29791c5326b 2024-11-07T15:53:49,319 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c9ea60f258db4fbaa3c4fd8689e5c60b 2024-11-07T15:53:49,320 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/aaacdfd392cc40bd915ea50365c4f0bd 2024-11-07T15:53:49,321 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2054f5d42ae7452a8b75eac39d7966f4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2054f5d42ae7452a8b75eac39d7966f4 2024-11-07T15:53:49,323 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6f47a77277d241979b1ded88dd65955c 2024-11-07T15:53:49,324 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/9fb53077d8ab4982b04f5025f6e73b0c 2024-11-07T15:53:49,325 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/c7b0a726c15e43aa9c95e1fd5f388da1 2024-11-07T15:53:49,326 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3130f181a4cb480689d4fd8e2aaf042b 2024-11-07T15:53:49,327 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d74c9110396e41e9b82da267db5d5a1e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/d74c9110396e41e9b82da267db5d5a1e 2024-11-07T15:53:49,328 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/e00f598ef588447f8b210a7f140c0718 2024-11-07T15:53:49,329 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/16c1009fff4b4273a97d475301c089cb 2024-11-07T15:53:49,330 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5517da70794641a9a04ff1dc209fcfef to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/5517da70794641a9a04ff1dc209fcfef 2024-11-07T15:53:49,331 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/53658cf287d54037bbf4546cb2b3a1d8 2024-11-07T15:53:49,332 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3a0e8562512c4627ba0cdca2ad659c08 2024-11-07T15:53:49,334 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/394698621be74801809bcfa0742ac7fd 2024-11-07T15:53:49,335 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2197e66384fa4dedbc8ca93854bf169c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2197e66384fa4dedbc8ca93854bf169c 2024-11-07T15:53:49,336 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/2ac1bf3a58ba4058b8b26d83d7dba966 2024-11-07T15:53:49,337 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/4a71df732b104bd9ba2684ee93503a3a 2024-11-07T15:53:49,338 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/466e366f65cc42928733f0bbe91cf20c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/466e366f65cc42928733f0bbe91cf20c 2024-11-07T15:53:49,339 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/6e4f8235bf914b07a6cccf9a369c68bb 2024-11-07T15:53:49,341 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/f3d9701b8bb64127a9531754d2d4ad36 2024-11-07T15:53:49,342 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/688435e9439b43a8b3f59fbe91d016bf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/688435e9439b43a8b3f59fbe91d016bf 2024-11-07T15:53:49,343 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/91646b0c5a434859b045525490790a51 2024-11-07T15:53:49,344 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc74edac35a4fd5b2aa7e656366756d 2024-11-07T15:53:49,345 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/587b3e01e2a04aa49828199c235e8e17 2024-11-07T15:53:49,347 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/dc8c6d3f6647417eb80a8901e46e8336, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/fe7f50bd023c44bcb41ee0407ab5405a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/c3e50bb3b28b4aa1b246331965823529, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e256434bc55c48218b8fe278e74978d6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e81e9cee417a4a9ba972079324c77a1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5b93a746e8ac427f9b666bf474fa94f8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6184fea0153145568ece3b97dcf4ec1e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/93570c3e7eb34ab2a1617411b5251e1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d772f2192d854394b2a3dbabd8a7a6b8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657] to archive 2024-11-07T15:53:49,348 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:53:49,349 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/611fd72873044363a120822df3f86f5a 2024-11-07T15:53:49,350 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/9888d605d7894bedb0e88efd03a23448 2024-11-07T15:53:49,352 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1358d18a56364cf38d7138a3be1d988d 2024-11-07T15:53:49,353 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/dc8c6d3f6647417eb80a8901e46e8336 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/dc8c6d3f6647417eb80a8901e46e8336 2024-11-07T15:53:49,354 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5a8c40ffb1c04c40beb4da80f2e767b2 2024-11-07T15:53:49,355 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c387e3855544e2da05f6cae61eb5690 2024-11-07T15:53:49,356 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4003694ca40a4779893b58e2c8b50c63 2024-11-07T15:53:49,357 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/fe7f50bd023c44bcb41ee0407ab5405a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/fe7f50bd023c44bcb41ee0407ab5405a 2024-11-07T15:53:49,358 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/106f77a614fb4dd3a4a07297c57b3ab8 2024-11-07T15:53:49,359 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b7e31497a46347bba586ba9a1d51c822 2024-11-07T15:53:49,360 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b610b2bd178849a9bed696aea3d1643c 2024-11-07T15:53:49,361 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/c3e50bb3b28b4aa1b246331965823529 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/c3e50bb3b28b4aa1b246331965823529 2024-11-07T15:53:49,362 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b1b5c955d8a84568a52e273e8830dc55 2024-11-07T15:53:49,363 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/30477f4a604f4dd8b1afda7ee5e531e4 2024-11-07T15:53:49,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e256434bc55c48218b8fe278e74978d6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e256434bc55c48218b8fe278e74978d6 2024-11-07T15:53:49,365 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/16a56521a55b43a6955a186960e9b885 2024-11-07T15:53:49,367 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e69ce9a0dbc1476e9efb037f220b92b5 2024-11-07T15:53:49,368 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/39758bb6f1c346d1b55ec6705ca216bd 2024-11-07T15:53:49,369 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/1eb734f7316c4805bd315155e577ceb5 2024-11-07T15:53:49,370 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e81e9cee417a4a9ba972079324c77a1f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e81e9cee417a4a9ba972079324c77a1f 2024-11-07T15:53:49,371 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/e840bb0b84394187b87fa1b947a58d91 2024-11-07T15:53:49,372 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/b204dd4610ad4a9bb8b19e7dd80dfc8d 2024-11-07T15:53:49,373 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5b93a746e8ac427f9b666bf474fa94f8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5b93a746e8ac427f9b666bf474fa94f8 2024-11-07T15:53:49,374 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/25d52f3f54e04be08c8e3a8da25a671b 2024-11-07T15:53:49,375 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/3816c32557254ca8bca70590f38b1d2e 2024-11-07T15:53:49,376 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/5c6ee2f8c1eb4a0980135d66c6feb173 2024-11-07T15:53:49,377 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6184fea0153145568ece3b97dcf4ec1e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6184fea0153145568ece3b97dcf4ec1e 2024-11-07T15:53:49,378 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/ba96ef846c934ee8a9ce9cffa7a95446 2024-11-07T15:53:49,379 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/af0a2b4f98544823a3e356fa23ff50a0 2024-11-07T15:53:49,380 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/93570c3e7eb34ab2a1617411b5251e1f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/93570c3e7eb34ab2a1617411b5251e1f 2024-11-07T15:53:49,382 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/f4777faac9d64f088802b1ead8943fd1 2024-11-07T15:53:49,383 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/7f3378788c08429da8f28c0bbf18457b 2024-11-07T15:53:49,384 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d772f2192d854394b2a3dbabd8a7a6b8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d772f2192d854394b2a3dbabd8a7a6b8 2024-11-07T15:53:49,385 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/6e3b3e5c76074533823d285c9df7540d 2024-11-07T15:53:49,386 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/26126382d3994ff1b9969b4bcaffb4d8 2024-11-07T15:53:49,387 DEBUG [StoreCloser-TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/64118f1a273d45dfb10404ca0cd74657 2024-11-07T15:53:49,392 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/recovered.edits/541.seqid, newMaxSeqId=541, maxSeqId=1 2024-11-07T15:53:49,392 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. 2024-11-07T15:53:49,393 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for b2da43428956e53a96aa3524187a4a97: Waiting for close lock at 1730994827968Running coprocessor pre-close hooks at 1730994827968Disabling compacts and flushes for region at 1730994827968Disabling writes for close at 1730994827968Obtaining lock to block concurrent updates at 1730994827968Preparing flush snapshotting stores in b2da43428956e53a96aa3524187a4a97 at 1730994827968Finished memstore snapshotting TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730994827969 (+1 ms)Flushing stores of TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97. at 1730994827969Flushing b2da43428956e53a96aa3524187a4a97/A: creating writer at 1730994827969Flushing b2da43428956e53a96aa3524187a4a97/A: appending metadata at 1730994827972 (+3 ms)Flushing b2da43428956e53a96aa3524187a4a97/A: closing flushed file at 1730994827972Flushing b2da43428956e53a96aa3524187a4a97/B: creating writer at 1730994828382 (+410 ms)Flushing b2da43428956e53a96aa3524187a4a97/B: appending metadata at 1730994828385 (+3 ms)Flushing b2da43428956e53a96aa3524187a4a97/B: closing flushed file at 1730994828386 (+1 ms)Flushing b2da43428956e53a96aa3524187a4a97/C: creating writer at 1730994828795 (+409 ms)Flushing b2da43428956e53a96aa3524187a4a97/C: appending metadata at 1730994828798 (+3 ms)Flushing b2da43428956e53a96aa3524187a4a97/C: closing flushed file at 1730994828798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d5e7ca9: reopening flushed file at 1730994829207 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba953ad: reopening flushed file at 1730994829213 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c19374b: reopening flushed file at 1730994829218 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for b2da43428956e53a96aa3524187a4a97 in 1256ms, sequenceid=538, compaction requested=false at 1730994829224 (+6 ms)Writing region close event to WAL at 1730994829388 (+164 ms)Running coprocessor post-close hooks at 1730994829392 (+4 ms)Closed at 1730994829392 2024-11-07T15:53:49,395 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:49,395 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=b2da43428956e53a96aa3524187a4a97, regionState=CLOSED 2024-11-07T15:53:49,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:53:49,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-07T15:53:49,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure b2da43428956e53a96aa3524187a4a97, server=c2682f91c1e2,42309,1730994796599 in 1.5870 sec 2024-11-07T15:53:49,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-07T15:53:49,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b2da43428956e53a96aa3524187a4a97, UNASSIGN in 1.5940 sec 2024-11-07T15:53:49,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-07T15:53:49,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6010 sec 2024-11-07T15:53:49,405 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994829405"}]},"ts":"1730994829405"} 2024-11-07T15:53:49,406 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:53:49,406 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:53:49,408 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6150 sec 2024-11-07T15:53:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-07T15:53:49,943 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:49,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:53:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:49,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-07T15:53:49,953 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:49,958 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:49,962 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/recovered.edits] 2024-11-07T15:53:49,965 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0033c8b3ef7543f3a043e9438f8587e7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/0033c8b3ef7543f3a043e9438f8587e7 2024-11-07T15:53:49,966 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e9958272fc724bba84d81cc28fbedd93 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/A/e9958272fc724bba84d81cc28fbedd93 2024-11-07T15:53:49,968 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc06552c0b64e88bdfce2ee42bf113b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/3cc06552c0b64e88bdfce2ee42bf113b 2024-11-07T15:53:49,969 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/a26b41b3e55143e9bf9e98e1c1b149f1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/B/a26b41b3e55143e9bf9e98e1c1b149f1 2024-11-07T15:53:49,972 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4512e5bf512b415387de4fb7f940bc42 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/4512e5bf512b415387de4fb7f940bc42 2024-11-07T15:53:49,973 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d5488d104eae4308896f7fea09379060 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/C/d5488d104eae4308896f7fea09379060 2024-11-07T15:53:49,976 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/recovered.edits/541.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97/recovered.edits/541.seqid 2024-11-07T15:53:49,977 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/b2da43428956e53a96aa3524187a4a97 2024-11-07T15:53:49,977 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:53:49,982 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:49,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-07T15:53:49,992 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:53:50,021 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:53:50,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:50,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:53:50,023 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994830023"}]},"ts":"9223372036854775807"} 2024-11-07T15:53:50,028 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:53:50,028 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b2da43428956e53a96aa3524187a4a97, NAME => 'TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:53:50,028 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:53:50,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994830028"}]},"ts":"9223372036854775807"} 2024-11-07T15:53:50,032 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:53:50,033 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:50,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 86 msec 2024-11-07T15:53:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-07T15:53:50,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:53:50,064 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:50,076 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=236 (was 210) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c2682f91c1e2:42309-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_816498657_22 at /127.0.0.1:56496 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=212 (was 120) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5781 (was 6293) 2024-11-07T15:53:50,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=236, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=212, ProcessCount=11, AvailableMemoryMB=5781 2024-11-07T15:53:50,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:53:50,086 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:53:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:50,088 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:53:50,088 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:50,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-07T15:53:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:50,089 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:53:50,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741951_1127 (size=959) 2024-11-07T15:53:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:50,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:50,498 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:53:50,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741952_1128 (size=53) 2024-11-07T15:53:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:50,905 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:50,905 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7dbbcb54c8468d54b4ae60ec53261963, disabling compactions & flushes 2024-11-07T15:53:50,905 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:50,905 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:50,905 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. after waiting 0 ms 2024-11-07T15:53:50,905 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:50,906 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:50,906 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7dbbcb54c8468d54b4ae60ec53261963: Waiting for close lock at 1730994830905Disabling compacts and flushes for region at 1730994830905Disabling writes for close at 1730994830905Writing region close event to WAL at 1730994830906 (+1 ms)Closed at 1730994830906 2024-11-07T15:53:50,907 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:53:50,907 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994830907"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994830907"}]},"ts":"1730994830907"} 2024-11-07T15:53:50,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:53:50,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:53:50,910 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994830910"}]},"ts":"1730994830910"} 2024-11-07T15:53:50,911 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:53:50,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, ASSIGN}] 2024-11-07T15:53:50,913 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, ASSIGN 2024-11-07T15:53:50,914 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:53:51,064 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:51,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, ASSIGN because future has completed 2024-11-07T15:53:51,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:51,223 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:51,223 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:53:51,224 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,224 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:51,224 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,224 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,225 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,227 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName A 2024-11-07T15:53:51,227 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:51,228 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:51,228 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,229 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName B 2024-11-07T15:53:51,229 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:51,230 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:51,230 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,231 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName C 2024-11-07T15:53:51,231 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:51,232 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:51,232 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,232 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,233 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,234 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,234 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,234 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:53:51,235 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,237 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:53:51,238 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 7dbbcb54c8468d54b4ae60ec53261963; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60547150, jitterRate=-0.09777715802192688}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:53:51,238 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:51,238 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 7dbbcb54c8468d54b4ae60ec53261963: Running coprocessor pre-open hook at 1730994831224Writing region info on filesystem at 1730994831224Initializing all the Stores at 1730994831225 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994831225Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994831225Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994831225Cleaning up temporary data from old regions at 1730994831234 (+9 ms)Running coprocessor post-open hooks at 1730994831238 (+4 ms)Region opened successfully at 1730994831238 2024-11-07T15:53:51,239 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., pid=36, masterSystemTime=1730994831219 2024-11-07T15:53:51,241 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:51,241 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:51,242 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:51,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:53:51,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-07T15:53:51,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 in 177 msec 2024-11-07T15:53:51,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-07T15:53:51,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, ASSIGN in 334 msec 2024-11-07T15:53:51,249 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:53:51,249 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994831249"}]},"ts":"1730994831249"} 2024-11-07T15:53:51,251 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:53:51,251 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:53:51,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-07T15:53:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-07T15:53:52,234 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:52,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:52,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:52,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:52,237 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d4aba1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:52,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:52,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:52,239 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49858, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b5f06a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:52,240 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:52,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:52,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:52,243 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:53:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:53:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b28b48a 2024-11-07T15:53:52,244 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:53:52,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:53:52,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:53:52,250 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:53:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:53:52,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741953_1129 (size=995) 2024-11-07T15:53:52,670 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:53:52,670 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:53:52,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:53:52,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE}] 2024-11-07T15:53:52,681 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE 2024-11-07T15:53:52,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:52,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE because future has completed 2024-11-07T15:53:52,683 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:53:52,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:52,836 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:52,836 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:53:52,836 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 7dbbcb54c8468d54b4ae60ec53261963, disabling compactions & flushes 2024-11-07T15:53:52,836 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:52,836 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:52,836 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. after waiting 0 ms 2024-11-07T15:53:52,836 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:52,840 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:53:52,841 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:52,841 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 7dbbcb54c8468d54b4ae60ec53261963: Waiting for close lock at 1730994832836Running coprocessor pre-close hooks at 1730994832836Disabling compacts and flushes for region at 1730994832836Disabling writes for close at 1730994832836Writing region close event to WAL at 1730994832837 (+1 ms)Running coprocessor post-close hooks at 1730994832841 (+4 ms)Closed at 1730994832841 2024-11-07T15:53:52,841 WARN [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 7dbbcb54c8468d54b4ae60ec53261963 to self. 2024-11-07T15:53:52,843 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:52,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=CLOSED 2024-11-07T15:53:52,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:53:52,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-07T15:53:52,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 in 163 msec 2024-11-07T15:53:52,849 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE; state=CLOSED, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=true 2024-11-07T15:53:52,999 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE because future has completed 2024-11-07T15:53:53,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:53:53,156 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,157 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:53:53,157 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,157 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:53:53,157 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,157 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,162 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,167 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName A 2024-11-07T15:53:53,169 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:53,169 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:53,170 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,170 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName B 2024-11-07T15:53:53,171 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:53,171 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:53,171 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,172 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dbbcb54c8468d54b4ae60ec53261963 columnFamilyName C 2024-11-07T15:53:53,172 DEBUG [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:53,172 INFO [StoreOpener-7dbbcb54c8468d54b4ae60ec53261963-1 {}] regionserver.HStore(327): Store=7dbbcb54c8468d54b4ae60ec53261963/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:53:53,172 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,173 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,174 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,175 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,175 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,175 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:53:53,177 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,177 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 7dbbcb54c8468d54b4ae60ec53261963; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61690614, jitterRate=-0.08073821663856506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:53:53,178 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,179 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 7dbbcb54c8468d54b4ae60ec53261963: Running coprocessor pre-open hook at 1730994833157Writing region info on filesystem at 1730994833157Initializing all the Stores at 1730994833158 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994833158Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994833158Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994833158Cleaning up temporary data from old regions at 1730994833175 (+17 ms)Running coprocessor post-open hooks at 1730994833178 (+3 ms)Region opened successfully at 1730994833179 (+1 ms) 2024-11-07T15:53:53,180 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., pid=41, masterSystemTime=1730994833153 2024-11-07T15:53:53,182 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,182 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=OPEN, openSeqNum=5, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:53:53,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-07T15:53:53,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 in 184 msec 2024-11-07T15:53:53,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-07T15:53:53,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, REOPEN/MOVE in 506 msec 2024-11-07T15:53:53,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-07T15:53:53,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-07T15:53:53,194 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-07T15:53:53,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-07T15:53:53,196 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:53,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,197 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,198 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60aa1498, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,199 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,200 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f475610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,202 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb134f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,202 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,203 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf0f230, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,205 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,205 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,205 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33aacc03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,206 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,207 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@536769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f208b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629b2e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e4c1660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,213 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ab6e00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,215 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,215 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f5a87e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,217 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742799f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,218 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,219 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab8384b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,221 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ecf1c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,223 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfcc4b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,223 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,224 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50014, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711709c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,225 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:53:53,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:53:53,226 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:53:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:53:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:53:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cb09b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:53:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:53:53,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:53:53,227 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:53:53,228 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6490be12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:53:53,229 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,229 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,229 DEBUG [Thread-660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,229 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,229 DEBUG [Thread-666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,229 DEBUG [Thread-662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,230 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,230 DEBUG [Thread-655 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,231 DEBUG [Thread-664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,231 DEBUG [Thread-655 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,232 DEBUG [Thread-649 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,232 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:53:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:53:53,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:53:53,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-07T15:53:53,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,234 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:53:53,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:53,235 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:53,236 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:53,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:53,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:53:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,254 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:53,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:53:53,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107156f1b76748843cca58154363f3e63e3_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994833251/Put/seqid=0 2024-11-07T15:53:53,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741954_1130 (size=14594) 2024-11-07T15:53:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:53,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994893331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994893332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994893333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994893334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994893348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994893454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994893454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994893454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994893455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994893464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994893673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994893673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994893674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994893674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:53,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994893674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:53,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:53,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,713 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:53,720 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107156f1b76748843cca58154363f3e63e3_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107156f1b76748843cca58154363f3e63e3_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54213e53c8ad443fa1136b8fa66a9d68, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:53,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54213e53c8ad443fa1136b8fa66a9d68 is 175, key is test_row_0/A:col10/1730994833251/Put/seqid=0 2024-11-07T15:53:53,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741955_1131 (size=39549) 2024-11-07T15:53:53,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54213e53c8ad443fa1136b8fa66a9d68 2024-11-07T15:53:53,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d3c8096100244b05a5723b01bc0d7b36 is 50, key is test_row_0/B:col10/1730994833251/Put/seqid=0 2024-11-07T15:53:53,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741956_1132 (size=12001) 2024-11-07T15:53:53,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d3c8096100244b05a5723b01bc0d7b36 2024-11-07T15:53:53,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/748e2486e37c48ef87c4bc33425d7ccd is 50, key is test_row_0/C:col10/1730994833251/Put/seqid=0 2024-11-07T15:53:53,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741957_1133 (size=12001) 2024-11-07T15:53:53,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/748e2486e37c48ef87c4bc33425d7ccd 2024-11-07T15:53:53,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:53,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54213e53c8ad443fa1136b8fa66a9d68 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68 2024-11-07T15:53:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:53,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:53,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68, entries=200, sequenceid=15, filesize=38.6 K 2024-11-07T15:53:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d3c8096100244b05a5723b01bc0d7b36 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36 2024-11-07T15:53:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:53,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:53:53,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/748e2486e37c48ef87c4bc33425d7ccd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd 2024-11-07T15:53:53,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd, entries=150, sequenceid=15, filesize=11.7 K 2024-11-07T15:53:53,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7dbbcb54c8468d54b4ae60ec53261963 in 620ms, sequenceid=15, compaction requested=false 2024-11-07T15:53:53,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:53,988 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:53,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:53:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994893994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994893997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:54,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:54,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:54,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:54,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:54,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:54,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:54,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110786c2baa9e5b842a09f5cfd991a911d05_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994833286/Put/seqid=0 2024-11-07T15:53:54,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994893999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741958_1134 (size=14594) 2024-11-07T15:53:54,031 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:54,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110786c2baa9e5b842a09f5cfd991a911d05_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786c2baa9e5b842a09f5cfd991a911d05_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:54,037 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a9c97cb8b1394c8f970711ce209c86d4, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:54,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a9c97cb8b1394c8f970711ce209c86d4 is 175, key is test_row_0/A:col10/1730994833286/Put/seqid=0 2024-11-07T15:53:54,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741959_1135 (size=39549) 2024-11-07T15:53:54,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a9c97cb8b1394c8f970711ce209c86d4 2024-11-07T15:53:54,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/13271af13ddb40eebd3ecfd33b65b570 is 50, key is test_row_0/B:col10/1730994833286/Put/seqid=0 2024-11-07T15:53:54,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741960_1136 (size=12001) 2024-11-07T15:53:54,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/13271af13ddb40eebd3ecfd33b65b570 2024-11-07T15:53:54,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/49a27671dd7a45e6b9e3020499748aa3 is 50, key is test_row_0/C:col10/1730994833286/Put/seqid=0 2024-11-07T15:53:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741961_1137 (size=12001) 2024-11-07T15:53:54,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994894114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994894114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/49a27671dd7a45e6b9e3020499748aa3 2024-11-07T15:53:54,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994894123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a9c97cb8b1394c8f970711ce209c86d4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4 2024-11-07T15:53:54,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4, entries=200, sequenceid=41, filesize=38.6 K 2024-11-07T15:53:54,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/13271af13ddb40eebd3ecfd33b65b570 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570 2024-11-07T15:53:54,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T15:53:54,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/49a27671dd7a45e6b9e3020499748aa3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3 2024-11-07T15:53:54,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T15:53:54,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 7dbbcb54c8468d54b4ae60ec53261963 in 168ms, sequenceid=41, compaction requested=false 2024-11-07T15:53:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:54,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-07T15:53:54,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:54,159 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T15:53:54,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107dda4fca6a6bd4cd8a3e54f84db4f13c6_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994833997/Put/seqid=0 2024-11-07T15:53:54,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741962_1138 (size=12154) 2024-11-07T15:53:54,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:54,192 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107dda4fca6a6bd4cd8a3e54f84db4f13c6_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dda4fca6a6bd4cd8a3e54f84db4f13c6_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/938a613eb3b44a36a376996871730773, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:54,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/938a613eb3b44a36a376996871730773 is 175, key is test_row_0/A:col10/1730994833997/Put/seqid=0 2024-11-07T15:53:54,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741963_1139 (size=30955) 2024-11-07T15:53:54,205 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/938a613eb3b44a36a376996871730773 2024-11-07T15:53:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/cd01448a88994613964fbe0951f99faf is 50, key is test_row_0/B:col10/1730994833997/Put/seqid=0 2024-11-07T15:53:54,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741964_1140 (size=12001) 2024-11-07T15:53:54,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994894366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994894371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994894373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,383 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994894483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,486 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994894484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994894493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,641 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/cd01448a88994613964fbe0951f99faf 2024-11-07T15:53:54,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/e3eba4d66d7f41c88db65f73a3e13e1a is 50, key is test_row_0/C:col10/1730994833997/Put/seqid=0 2024-11-07T15:53:54,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741965_1141 (size=12001) 2024-11-07T15:53:54,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/e3eba4d66d7f41c88db65f73a3e13e1a 2024-11-07T15:53:54,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/938a613eb3b44a36a376996871730773 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773 2024-11-07T15:53:54,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773, entries=150, sequenceid=51, filesize=30.2 K 2024-11-07T15:53:54,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/cd01448a88994613964fbe0951f99faf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf 2024-11-07T15:53:54,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T15:53:54,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/e3eba4d66d7f41c88db65f73a3e13e1a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a 2024-11-07T15:53:54,693 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T15:53:54,694 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 7dbbcb54c8468d54b4ae60ec53261963 in 535ms, sequenceid=51, compaction requested=true 2024-11-07T15:53:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:54,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-07T15:53:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-07T15:53:54,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-07T15:53:54,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-07T15:53:54,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.4690 sec 2024-11-07T15:53:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:54,704 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:54,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T15:53:54,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107caeeba9f646341aba0ab87d3e17a4e65_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:54,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741966_1142 (size=14594) 2024-11-07T15:53:54,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994894708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994894714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,763 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994894762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994894873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994894874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994894874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994894874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994894874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:54,952 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:53:55,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994895084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994895084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994895084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994895093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,096 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994895093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,130 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,136 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107caeeba9f646341aba0ab87d3e17a4e65_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107caeeba9f646341aba0ab87d3e17a4e65_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:55,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/496b04648e8c48539f522fbe0f6a42ba, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/496b04648e8c48539f522fbe0f6a42ba is 175, key is test_row_0/A:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:55,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741967_1143 (size=39549) 2024-11-07T15:53:55,147 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/496b04648e8c48539f522fbe0f6a42ba 2024-11-07T15:53:55,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d8438e60d0bd4ae8bcd89da8080d4846 is 50, key is test_row_0/B:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:55,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741968_1144 (size=12001) 2024-11-07T15:53:55,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d8438e60d0bd4ae8bcd89da8080d4846 2024-11-07T15:53:55,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/61b0c1da538d47418dc6a14677ff3220 is 50, key is test_row_0/C:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:55,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741969_1145 (size=12001) 2024-11-07T15:53:55,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-07T15:53:55,394 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:55,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-07T15:53:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:55,396 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:55,397 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:55,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994895393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994895393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994895394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994895403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994895403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:55,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:53:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:55,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:55,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:55,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:55,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/61b0c1da538d47418dc6a14677ff3220 2024-11-07T15:53:55,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/496b04648e8c48539f522fbe0f6a42ba as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba 2024-11-07T15:53:55,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba, entries=200, sequenceid=81, filesize=38.6 K 2024-11-07T15:53:55,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d8438e60d0bd4ae8bcd89da8080d4846 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846 2024-11-07T15:53:55,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T15:53:55,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/61b0c1da538d47418dc6a14677ff3220 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220 2024-11-07T15:53:55,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220, entries=150, sequenceid=81, filesize=11.7 K 2024-11-07T15:53:55,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 7dbbcb54c8468d54b4ae60ec53261963 in 910ms, sequenceid=81, compaction requested=true 2024-11-07T15:53:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:55,614 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:55,615 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:55,615 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,616 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:55,616 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:53:55,617 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,617 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:55,617 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=146.1 K 2024-11-07T15:53:55,617 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:53:55,617 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,617 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,617 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba] 2024-11-07T15:53:55,617 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=46.9 K 2024-11-07T15:53:55,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d3c8096100244b05a5723b01bc0d7b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730994833243 2024-11-07T15:53:55,624 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54213e53c8ad443fa1136b8fa66a9d68, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730994833243 2024-11-07T15:53:55,624 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9c97cb8b1394c8f970711ce209c86d4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994833286 2024-11-07T15:53:55,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 13271af13ddb40eebd3ecfd33b65b570, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994833286 2024-11-07T15:53:55,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,625 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 938a613eb3b44a36a376996871730773, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994833995 2024-11-07T15:53:55,625 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting cd01448a88994613964fbe0951f99faf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994833995 2024-11-07T15:53:55,625 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 496b04648e8c48539f522fbe0f6a42ba, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,626 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d8438e60d0bd4ae8bcd89da8080d4846, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,648 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,653 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107aa0a1c53e6d7429d98939099f43a6a0f_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,656 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#127 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:55,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,657 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/09e34fadfa8c4a0c9870fc230952328a is 50, key is test_row_0/B:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,661 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107aa0a1c53e6d7429d98939099f43a6a0f_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,662 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107aa0a1c53e6d7429d98939099f43a6a0f_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741970_1146 (size=12139) 2024-11-07T15:53:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741971_1147 (size=4469) 2024-11-07T15:53:55,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,700 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/09e34fadfa8c4a0c9870fc230952328a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/09e34fadfa8c4a0c9870fc230952328a 2024-11-07T15:53:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-07T15:53:55,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,706 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:53:55,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,708 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 09e34fadfa8c4a0c9870fc230952328a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:55,708 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:55,708 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=12, startTime=1730994835614; duration=0sec 2024-11-07T15:53:55,708 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:55,708 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:53:55,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,708 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:53:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,710 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:53:55,711 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:53:55,711 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:55,711 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=46.9 K 2024-11-07T15:53:55,711 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 748e2486e37c48ef87c4bc33425d7ccd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1730994833243 2024-11-07T15:53:55,712 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 49a27671dd7a45e6b9e3020499748aa3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994833286 2024-11-07T15:53:55,713 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e3eba4d66d7f41c88db65f73a3e13e1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994833995 2024-11-07T15:53:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:55,713 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b0c1da538d47418dc6a14677ff3220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757acf7982e184c66b4b5759459115b0a_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994834715/Put/seqid=0 2024-11-07T15:53:55,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,735 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#129 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,735 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/70e0e6ade38a442cbdb2119153a5955a is 50, key is test_row_0/C:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741972_1148 (size=9714) 2024-11-07T15:53:55,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,757 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110757acf7982e184c66b4b5759459115b0a_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757acf7982e184c66b4b5759459115b0a_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a3e693bc936a42028d9bdec55b149f07, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a3e693bc936a42028d9bdec55b149f07 is 175, key is test_row_0/A:col10/1730994834715/Put/seqid=0 2024-11-07T15:53:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741973_1149 (size=12139) 2024-11-07T15:53:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741974_1150 (size=22361) 2024-11-07T15:53:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:55,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:55,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994895963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994895963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994895969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994895971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994895971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:55,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:56,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994896083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994896084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,092 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994896084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994896084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994896084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,097 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#126 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:56,097 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/2ffca5a2e8ec4278986e3c2b4ffca667 is 175, key is test_row_0/A:col10/1730994834370/Put/seqid=0 2024-11-07T15:53:56,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741975_1151 (size=31093) 2024-11-07T15:53:56,122 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/2ffca5a2e8ec4278986e3c2b4ffca667 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667 2024-11-07T15:53:56,132 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 2ffca5a2e8ec4278986e3c2b4ffca667(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:56,132 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:56,132 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=12, startTime=1730994835614; duration=0sec 2024-11-07T15:53:56,132 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:56,132 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:53:56,175 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/70e0e6ade38a442cbdb2119153a5955a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/70e0e6ade38a442cbdb2119153a5955a 2024-11-07T15:53:56,182 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 70e0e6ade38a442cbdb2119153a5955a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:56,182 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:56,182 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=12, startTime=1730994835615; duration=0sec 2024-11-07T15:53:56,182 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:56,183 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:53:56,187 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a3e693bc936a42028d9bdec55b149f07 2024-11-07T15:53:56,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/85b0c4682eba458fa82cb99c1f779b6e is 50, key is test_row_0/B:col10/1730994834715/Put/seqid=0 2024-11-07T15:53:56,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741976_1152 (size=9657) 2024-11-07T15:53:56,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994896293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994896304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994896304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994896304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994896304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:56,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,611 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/85b0c4682eba458fa82cb99c1f779b6e 2024-11-07T15:53:56,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994896613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994896613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994896613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994896614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:56,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7c45295d95614b70b0a1df0ad349cca0 is 50, key is test_row_0/C:col10/1730994834715/Put/seqid=0 2024-11-07T15:53:56,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741977_1153 (size=9657) 2024-11-07T15:53:56,636 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7c45295d95614b70b0a1df0ad349cca0 2024-11-07T15:53:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/a3e693bc936a42028d9bdec55b149f07 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07 2024-11-07T15:53:56,651 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07, entries=100, sequenceid=89, filesize=21.8 K 2024-11-07T15:53:56,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/85b0c4682eba458fa82cb99c1f779b6e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e 2024-11-07T15:53:56,660 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e, entries=100, sequenceid=89, filesize=9.4 K 2024-11-07T15:53:56,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7c45295d95614b70b0a1df0ad349cca0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0 2024-11-07T15:53:56,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0, entries=100, sequenceid=89, filesize=9.4 K 2024-11-07T15:53:56,671 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 7dbbcb54c8468d54b4ae60ec53261963 in 966ms, sequenceid=89, compaction requested=false 2024-11-07T15:53:56,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:56,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:56,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-07T15:53:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-07T15:53:56,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-07T15:53:56,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-07T15:53:56,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2820 sec 2024-11-07T15:53:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:57,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:57,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T15:53:57,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073364da2bc03f44b08d0469d6bb92bfa6_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994897123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994897123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994897123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741978_1154 (size=14594) 2024-11-07T15:53:57,146 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:57,152 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411073364da2bc03f44b08d0469d6bb92bfa6_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073364da2bc03f44b08d0469d6bb92bfa6_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:57,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e06947690de047f492372c8fc5337362, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e06947690de047f492372c8fc5337362 is 175, key is test_row_0/A:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741979_1155 (size=39549) 2024-11-07T15:53:57,171 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e06947690de047f492372c8fc5337362 2024-11-07T15:53:57,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/411abd592b1347d7b4279720b00fb75d is 50, key is test_row_0/B:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741980_1156 (size=12001) 2024-11-07T15:53:57,202 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/411abd592b1347d7b4279720b00fb75d 2024-11-07T15:53:57,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/f8b2b6c184bd49419678d863c8bb4fe9 is 50, key is test_row_0/C:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741981_1157 (size=12001) 2024-11-07T15:53:57,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/f8b2b6c184bd49419678d863c8bb4fe9 2024-11-07T15:53:57,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e06947690de047f492372c8fc5337362 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362 2024-11-07T15:53:57,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362, entries=200, sequenceid=121, filesize=38.6 K 2024-11-07T15:53:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/411abd592b1347d7b4279720b00fb75d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d 2024-11-07T15:53:57,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-07T15:53:57,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/f8b2b6c184bd49419678d863c8bb4fe9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9 2024-11-07T15:53:57,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9, entries=150, sequenceid=121, filesize=11.7 K 2024-11-07T15:53:57,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 7dbbcb54c8468d54b4ae60ec53261963 in 164ms, sequenceid=121, compaction requested=true 2024-11-07T15:53:57,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:57,278 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:57,279 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:57,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:57,280 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:57,280 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:53:57,280 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,280 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=90.8 K 2024-11-07T15:53:57,280 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,280 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362] 2024-11-07T15:53:57,280 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:57,280 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:53:57,280 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,281 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/09e34fadfa8c4a0c9870fc230952328a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=33.0 K 2024-11-07T15:53:57,281 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffca5a2e8ec4278986e3c2b4ffca667, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:57,281 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e34fadfa8c4a0c9870fc230952328a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:57,282 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e693bc936a42028d9bdec55b149f07, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730994834715 2024-11-07T15:53:57,282 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85b0c4682eba458fa82cb99c1f779b6e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730994834715 2024-11-07T15:53:57,282 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e06947690de047f492372c8fc5337362, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:57,283 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411abd592b1347d7b4279720b00fb75d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:57,295 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,296 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:57,297 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/891533516ff340a1ac09e5613b8d177a is 50, key is test_row_0/B:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,299 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107167db16cc2f844e0923f81a338b579b9_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,301 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107167db16cc2f844e0923f81a338b579b9_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,302 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107167db16cc2f844e0923f81a338b579b9_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741982_1158 (size=12241) 2024-11-07T15:53:57,320 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/891533516ff340a1ac09e5613b8d177a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/891533516ff340a1ac09e5613b8d177a 2024-11-07T15:53:57,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741983_1159 (size=4469) 2024-11-07T15:53:57,325 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#136 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:57,325 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/26815ebf6d6e4eb4873e5465d6529f42 is 175, key is test_row_0/A:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,330 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 891533516ff340a1ac09e5613b8d177a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:57,330 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:57,330 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=13, startTime=1730994837279; duration=0sec 2024-11-07T15:53:57,330 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:57,330 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:53:57,330 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:57,332 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:57,332 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:53:57,332 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,333 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/70e0e6ade38a442cbdb2119153a5955a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=33.0 K 2024-11-07T15:53:57,333 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70e0e6ade38a442cbdb2119153a5955a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1730994834370 2024-11-07T15:53:57,334 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c45295d95614b70b0a1df0ad349cca0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1730994834715 2024-11-07T15:53:57,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741984_1160 (size=31195) 2024-11-07T15:53:57,336 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b2b6c184bd49419678d863c8bb4fe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:57,350 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:57,350 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/26815ebf6d6e4eb4873e5465d6529f42 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42 2024-11-07T15:53:57,351 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/8889dea86d0f40ffa5a57c98f60ae02d is 50, key is test_row_0/C:col10/1730994835960/Put/seqid=0 2024-11-07T15:53:57,358 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 26815ebf6d6e4eb4873e5465d6529f42(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:57,358 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:57,358 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=13, startTime=1730994837278; duration=0sec 2024-11-07T15:53:57,358 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:57,358 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:53:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741985_1161 (size=12241) 2024-11-07T15:53:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:57,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:57,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:53:57,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075fab5cb8de6a4560b9768dffd0127259_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994837458/Put/seqid=0 2024-11-07T15:53:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741986_1162 (size=12254) 2024-11-07T15:53:57,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:57,497 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075fab5cb8de6a4560b9768dffd0127259_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fab5cb8de6a4560b9768dffd0127259_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:57,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/c5c9172a428846a7a0f62f2a555993f3, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:57,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/c5c9172a428846a7a0f62f2a555993f3 is 175, key is test_row_0/A:col10/1730994837458/Put/seqid=0 2024-11-07T15:53:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741987_1163 (size=31055) 2024-11-07T15:53:57,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/c5c9172a428846a7a0f62f2a555993f3 2024-11-07T15:53:57,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/4175c0ffc58d44e198c0c77799f15b3b is 50, key is test_row_0/B:col10/1730994837458/Put/seqid=0 2024-11-07T15:53:57,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741988_1164 (size=12101) 2024-11-07T15:53:57,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/4175c0ffc58d44e198c0c77799f15b3b 2024-11-07T15:53:57,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-07T15:53:57,543 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:53:57,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:53:57,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-07T15:53:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:57,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:53:57,550 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:53:57,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:53:57,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ff403ae07c824df4a5140a08ac4170fb is 50, key is test_row_0/C:col10/1730994837458/Put/seqid=0 2024-11-07T15:53:57,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741989_1165 (size=12101) 2024-11-07T15:53:57,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ff403ae07c824df4a5140a08ac4170fb 2024-11-07T15:53:57,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/c5c9172a428846a7a0f62f2a555993f3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3 2024-11-07T15:53:57,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3, entries=150, sequenceid=134, filesize=30.3 K 2024-11-07T15:53:57,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/4175c0ffc58d44e198c0c77799f15b3b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b 2024-11-07T15:53:57,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b, entries=150, sequenceid=134, filesize=11.8 K 2024-11-07T15:53:57,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ff403ae07c824df4a5140a08ac4170fb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb 2024-11-07T15:53:57,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb, entries=150, sequenceid=134, filesize=11.8 K 2024-11-07T15:53:57,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7dbbcb54c8468d54b4ae60ec53261963 in 154ms, sequenceid=134, compaction requested=false 2024-11-07T15:53:57,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:57,648 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:53:57,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T15:53:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:57,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110772f9f6d355da45348c45b774d8182ae4_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:57,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741990_1166 (size=17284) 2024-11-07T15:53:57,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,776 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/8889dea86d0f40ffa5a57c98f60ae02d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/8889dea86d0f40ffa5a57c98f60ae02d 2024-11-07T15:53:57,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,783 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 8889dea86d0f40ffa5a57c98f60ae02d(size=12.0 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:57,783 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:57,783 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=13, startTime=1730994837279; duration=0sec 2024-11-07T15:53:57,783 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:57,783 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:53:57,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:57,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:57,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994897982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,984 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994897993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:58,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110772f9f6d355da45348c45b774d8182ae4_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772f9f6d355da45348c45b774d8182ae4_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:58,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d9d644860be4930a2b2477d4c436334, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:58,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d9d644860be4930a2b2477d4c436334 is 175, key is test_row_0/A:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:58,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741991_1167 (size=48389) 2024-11-07T15:53:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994898143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994898143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,150 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,151 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:58,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994898153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,154 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994898293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994898303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=162, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d9d644860be4930a2b2477d4c436334 2024-11-07T15:53:58,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/87a96e8ef1de4d8aa15bfad017c858f4 is 50, key is test_row_0/B:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:58,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741992_1168 (size=12151) 2024-11-07T15:53:58,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/87a96e8ef1de4d8aa15bfad017c858f4 2024-11-07T15:53:58,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/28cbc5b9bde648ba9dd07786c93f9e38 is 50, key is test_row_0/C:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:58,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741993_1169 (size=12151) 2024-11-07T15:53:58,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:58,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994898803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,808 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994898813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:58,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:58,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/28cbc5b9bde648ba9dd07786c93f9e38 2024-11-07T15:53:58,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d9d644860be4930a2b2477d4c436334 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334 2024-11-07T15:53:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:53:58,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334, entries=250, sequenceid=162, filesize=47.3 K 2024-11-07T15:53:58,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/87a96e8ef1de4d8aa15bfad017c858f4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4 2024-11-07T15:53:58,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4, entries=150, sequenceid=162, filesize=11.9 K 2024-11-07T15:53:58,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/28cbc5b9bde648ba9dd07786c93f9e38 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38 2024-11-07T15:53:58,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38, entries=150, sequenceid=162, filesize=11.9 K 2024-11-07T15:53:58,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 7dbbcb54c8468d54b4ae60ec53261963 in 1309ms, sequenceid=162, compaction requested=true 2024-11-07T15:53:58,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:53:58,958 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:53:58,958 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:58,959 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110639 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:58,959 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:58,959 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:53:58,959 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:53:58,959 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,959 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,959 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=108.0 K 2024-11-07T15:53:58,959 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/891533516ff340a1ac09e5613b8d177a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=35.6 K 2024-11-07T15:53:58,959 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:58,960 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334] 2024-11-07T15:53:58,961 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 891533516ff340a1ac09e5613b8d177a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:58,961 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26815ebf6d6e4eb4873e5465d6529f42, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:58,961 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4175c0ffc58d44e198c0c77799f15b3b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730994837127 2024-11-07T15:53:58,962 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c9172a428846a7a0f62f2a555993f3, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730994837127 2024-11-07T15:53:58,962 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a96e8ef1de4d8aa15bfad017c858f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:53:58,962 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9d644860be4930a2b2477d4c436334, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:53:58,972 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:58,975 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:58,975 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/86eb074732b241a7aec780fe6e243538 is 50, key is test_row_0/B:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:58,985 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b4a94be548f24f3cb303d6119f8de5f2_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:58,986 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b4a94be548f24f3cb303d6119f8de5f2_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:58,987 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b4a94be548f24f3cb303d6119f8de5f2_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:58,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741994_1170 (size=12493) 2024-11-07T15:53:58,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741995_1171 (size=4469) 2024-11-07T15:53:58,997 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#144 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:58,997 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eac2636413a14ff78fbd91bba07a7592 is 175, key is test_row_0/A:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:58,999 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/86eb074732b241a7aec780fe6e243538 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/86eb074732b241a7aec780fe6e243538 2024-11-07T15:53:59,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741996_1172 (size=31447) 2024-11-07T15:53:59,007 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 86eb074732b241a7aec780fe6e243538(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:59,007 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:59,007 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=13, startTime=1730994838958; duration=0sec 2024-11-07T15:53:59,007 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:53:59,007 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:53:59,007 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:53:59,008 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:53:59,008 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:53:59,008 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:59,008 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/8889dea86d0f40ffa5a57c98f60ae02d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=35.6 K 2024-11-07T15:53:59,009 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8889dea86d0f40ffa5a57c98f60ae02d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1730994835960 2024-11-07T15:53:59,010 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ff403ae07c824df4a5140a08ac4170fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1730994837127 2024-11-07T15:53:59,010 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 28cbc5b9bde648ba9dd07786c93f9e38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:53:59,012 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eac2636413a14ff78fbd91bba07a7592 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592 2024-11-07T15:53:59,020 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into eac2636413a14ff78fbd91bba07a7592(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:59,020 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:59,020 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=13, startTime=1730994838958; duration=0sec 2024-11-07T15:53:59,020 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:59,020 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:53:59,023 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:53:59,024 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/47a0f7cf14264af39a02bf24e6c7298f is 50, key is test_row_0/C:col10/1730994837647/Put/seqid=0 2024-11-07T15:53:59,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741997_1173 (size=12493) 2024-11-07T15:53:59,046 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/47a0f7cf14264af39a02bf24e6c7298f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/47a0f7cf14264af39a02bf24e6c7298f 2024-11-07T15:53:59,053 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 47a0f7cf14264af39a02bf24e6c7298f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:53:59,053 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:53:59,053 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=13, startTime=1730994838958; duration=0sec 2024-11-07T15:53:59,053 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:53:59,053 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:53:59,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-07T15:53:59,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:53:59,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:53:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e8642cb059f54656a4532568fc7a1bc3_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994837656/Put/seqid=0 2024-11-07T15:53:59,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741998_1174 (size=12304) 2024-11-07T15:53:59,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:53:59,113 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e8642cb059f54656a4532568fc7a1bc3_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e8642cb059f54656a4532568fc7a1bc3_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:59,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eb85e0da23a847719d806b74e7d61a5e, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:53:59,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eb85e0da23a847719d806b74e7d61a5e is 175, key is test_row_0/A:col10/1730994837656/Put/seqid=0 2024-11-07T15:53:59,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741999_1175 (size=31105) 2024-11-07T15:53:59,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eb85e0da23a847719d806b74e7d61a5e 2024-11-07T15:53:59,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/af771974de1948a5a23e929b5fb695bd is 50, key is test_row_0/B:col10/1730994837656/Put/seqid=0 2024-11-07T15:53:59,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742000_1176 (size=12151) 2024-11-07T15:53:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:53:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:53:59,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:53:59,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:59,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994899873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:59,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,877 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:59,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:59,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994899875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:59,937 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/af771974de1948a5a23e929b5fb695bd 2024-11-07T15:53:59,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7bd586eb3e7a431db230c9ea62aad683 is 50, key is test_row_0/C:col10/1730994837656/Put/seqid=0 2024-11-07T15:53:59,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742001_1177 (size=12151) 2024-11-07T15:53:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:59,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994899983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:59,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:53:59,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994899983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:53:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:53:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:53:59,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994900163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,166 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994900173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994900173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,176 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,177 INFO [master/c2682f91c1e2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,177 INFO [master/c2682f91c1e2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T15:54:00,176 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994900193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994900193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7bd586eb3e7a431db230c9ea62aad683 2024-11-07T15:54:00,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/eb85e0da23a847719d806b74e7d61a5e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e 2024-11-07T15:54:00,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e, entries=150, sequenceid=174, filesize=30.4 K 2024-11-07T15:54:00,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/af771974de1948a5a23e929b5fb695bd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd 2024-11-07T15:54:00,367 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T15:54:00,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7bd586eb3e7a431db230c9ea62aad683 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683 2024-11-07T15:54:00,373 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683, entries=150, sequenceid=174, filesize=11.9 K 2024-11-07T15:54:00,374 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 7dbbcb54c8468d54b4ae60ec53261963 in 1288ms, sequenceid=174, compaction requested=false 2024-11-07T15:54:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:00,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-07T15:54:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-07T15:54:00,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-07T15:54:00,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8260 sec 2024-11-07T15:54:00,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8350 sec 2024-11-07T15:54:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:00,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:00,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T15:54:00,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070869c3c9a0224a6082ecd86f5d9de098_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:00,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994900532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994900534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742002_1178 (size=17284) 2024-11-07T15:54:00,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994900643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994900643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,646 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,647 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994900853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994900853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:00,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:00,942 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:00,947 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070869c3c9a0224a6082ecd86f5d9de098_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070869c3c9a0224a6082ecd86f5d9de098_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:00,948 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/7b37e3bd37e145498506a39a1d42d549, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:00,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/7b37e3bd37e145498506a39a1d42d549 is 175, key is test_row_0/A:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742003_1179 (size=48389) 2024-11-07T15:54:00,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/7b37e3bd37e145498506a39a1d42d549 2024-11-07T15:54:00,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/954f2c3e2ee74724a3828366e61ea537 is 50, key is test_row_0/B:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:00,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742004_1180 (size=12151) 2024-11-07T15:54:00,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/954f2c3e2ee74724a3828366e61ea537 2024-11-07T15:54:00,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ffbf28c0383f44a9b68be98afc0d0f40 is 50, key is test_row_0/C:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:00,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742005_1181 (size=12151) 2024-11-07T15:54:01,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994901163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994901163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:01,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ffbf28c0383f44a9b68be98afc0d0f40 2024-11-07T15:54:01,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/7b37e3bd37e145498506a39a1d42d549 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549 2024-11-07T15:54:01,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549, entries=250, sequenceid=204, filesize=47.3 K 2024-11-07T15:54:01,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/954f2c3e2ee74724a3828366e61ea537 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537 2024-11-07T15:54:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T15:54:01,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/ffbf28c0383f44a9b68be98afc0d0f40 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40 2024-11-07T15:54:01,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40, entries=150, sequenceid=204, filesize=11.9 K 2024-11-07T15:54:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for 7dbbcb54c8468d54b4ae60ec53261963 in 901ms, sequenceid=204, compaction requested=true 2024-11-07T15:54:01,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:01,407 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:01,407 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:01,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,408 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:01,408 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:54:01,408 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,408 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/86eb074732b241a7aec780fe6e243538, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=35.9 K 2024-11-07T15:54:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,409 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 86eb074732b241a7aec780fe6e243538, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:54:01,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,410 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting af771974de1948a5a23e929b5fb695bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730994837656 2024-11-07T15:54:01,411 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 954f2c3e2ee74724a3828366e61ea537, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839874 2024-11-07T15:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:01,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:54:01,411 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,411 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=108.3 K 2024-11-07T15:54:01,411 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549] 2024-11-07T15:54:01,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,412 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting eac2636413a14ff78fbd91bba07a7592, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:54:01,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,413 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb85e0da23a847719d806b74e7d61a5e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730994837656 2024-11-07T15:54:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,414 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b37e3bd37e145498506a39a1d42d549, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839870 2024-11-07T15:54:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,426 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#153 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,427 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/775fa127d57d46189d0b3642791b0dba is 50, key is test_row_0/B:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,437 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,441 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107ea508fdfdeef4f059bd9e7c4d0f2221f_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:01,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742006_1182 (size=12595) 2024-11-07T15:54:01,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,449 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107ea508fdfdeef4f059bd9e7c4d0f2221f_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,449 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107ea508fdfdeef4f059bd9e7c4d0f2221f_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:01,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742007_1183 (size=4469) 2024-11-07T15:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:01,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:01,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107109f1a5e8d174e738e2fe947b0b2d2a9_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_1/A:col10/1730994841710/Put/seqid=0 2024-11-07T15:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-07T15:54:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,724 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-07T15:54:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,727 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:54:01,729 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742008_1184 (size=14794) 2024-11-07T15:54:01,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,743 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107109f1a5e8d174e738e2fe947b0b2d2a9_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107109f1a5e8d174e738e2fe947b0b2d2a9_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:01,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a61e4ec9be248e98a7948b963af526f, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a61e4ec9be248e98a7948b963af526f is 175, key is test_row_1/A:col10/1730994841710/Put/seqid=0 2024-11-07T15:54:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742009_1185 (size=39745) 2024-11-07T15:54:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,770 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a61e4ec9be248e98a7948b963af526f 2024-11-07T15:54:01,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:01,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/89ecb2d0e3c3460789626b8871ddc10d is 50, key is test_row_1/B:col10/1730994841710/Put/seqid=0 2024-11-07T15:54:01,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742010_1186 (size=9757) 2024-11-07T15:54:01,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/89ecb2d0e3c3460789626b8871ddc10d 2024-11-07T15:54:01,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/d6ce0eb3cdcd428c8d753acabab25f74 is 50, key is test_row_1/C:col10/1730994841710/Put/seqid=0 2024-11-07T15:54:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:54:01,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742011_1187 (size=9757) 2024-11-07T15:54:01,854 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/775fa127d57d46189d0b3642791b0dba as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/775fa127d57d46189d0b3642791b0dba 2024-11-07T15:54:01,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994901854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994901854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:01,862 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#154 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:01,863 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/6cd92e3929234f63b34cf493e11f8724 is 175, key is test_row_0/A:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:01,867 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 775fa127d57d46189d0b3642791b0dba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:01,867 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=13, startTime=1730994841407; duration=0sec 2024-11-07T15:54:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:54:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:01,868 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:01,869 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:54:01,869 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,869 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/47a0f7cf14264af39a02bf24e6c7298f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=35.9 K 2024-11-07T15:54:01,869 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a0f7cf14264af39a02bf24e6c7298f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1730994837507 2024-11-07T15:54:01,870 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bd586eb3e7a431db230c9ea62aad683, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1730994837656 2024-11-07T15:54:01,871 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbf28c0383f44a9b68be98afc0d0f40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839874 2024-11-07T15:54:01,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:54:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:01,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:01,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:01,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:01,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742012_1188 (size=31549) 2024-11-07T15:54:01,890 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:01,891 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/34664d4e814a44e990bae7c0ecbc2fba is 50, key is test_row_0/C:col10/1730994839874/Put/seqid=0 2024-11-07T15:54:01,898 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/6cd92e3929234f63b34cf493e11f8724 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724 2024-11-07T15:54:01,907 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 6cd92e3929234f63b34cf493e11f8724(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:01,907 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:01,907 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=13, startTime=1730994841406; duration=0sec 2024-11-07T15:54:01,907 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:01,907 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:54:01,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742013_1189 (size=12595) 2024-11-07T15:54:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994901964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994901964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:54:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:54:02,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994902173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994902173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:54:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:02,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/d6ce0eb3cdcd428c8d753acabab25f74 2024-11-07T15:54:02,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a61e4ec9be248e98a7948b963af526f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f 2024-11-07T15:54:02,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f, entries=200, sequenceid=215, filesize=38.8 K 2024-11-07T15:54:02,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/89ecb2d0e3c3460789626b8871ddc10d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d 2024-11-07T15:54:02,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d, entries=100, sequenceid=215, filesize=9.5 K 2024-11-07T15:54:02,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/d6ce0eb3cdcd428c8d753acabab25f74 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74 2024-11-07T15:54:02,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74, entries=100, sequenceid=215, filesize=9.5 K 2024-11-07T15:54:02,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7dbbcb54c8468d54b4ae60ec53261963 in 571ms, sequenceid=215, compaction requested=false 2024-11-07T15:54:02,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:02,329 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/34664d4e814a44e990bae7c0ecbc2fba as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/34664d4e814a44e990bae7c0ecbc2fba 2024-11-07T15:54:02,335 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 34664d4e814a44e990bae7c0ecbc2fba(size=12.3 K), total size for store is 21.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:02,335 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:02,335 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=13, startTime=1730994841407; duration=0sec 2024-11-07T15:54:02,335 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:02,335 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:54:02,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-07T15:54:02,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:54:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:54:02,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110765db9de9d529498f8ff9842c9c76dd41_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994841808/Put/seqid=0 2024-11-07T15:54:02,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742014_1190 (size=12304) 2024-11-07T15:54:02,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,375 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110765db9de9d529498f8ff9842c9c76dd41_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765db9de9d529498f8ff9842c9c76dd41_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a1dba93e05c45aa9a8379487f9ba3d9, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:02,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a1dba93e05c45aa9a8379487f9ba3d9 is 175, key is test_row_0/A:col10/1730994841808/Put/seqid=0 2024-11-07T15:54:02,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742015_1191 (size=31105) 2024-11-07T15:54:02,393 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a1dba93e05c45aa9a8379487f9ba3d9 2024-11-07T15:54:02,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/6bcbc12ced104e84ba8432f04dff7894 is 50, key is test_row_0/B:col10/1730994841808/Put/seqid=0 2024-11-07T15:54:02,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742016_1192 (size=12151) 2024-11-07T15:54:02,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/6bcbc12ced104e84ba8432f04dff7894 2024-11-07T15:54:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7155bd4b0e53419885801dcc25bb3281 is 50, key is test_row_0/C:col10/1730994841808/Put/seqid=0 2024-11-07T15:54:02,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742017_1193 (size=12151) 2024-11-07T15:54:02,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7155bd4b0e53419885801dcc25bb3281 2024-11-07T15:54:02,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/1a1dba93e05c45aa9a8379487f9ba3d9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9 2024-11-07T15:54:02,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9, entries=150, sequenceid=243, filesize=30.4 K 2024-11-07T15:54:02,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/6bcbc12ced104e84ba8432f04dff7894 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894 2024-11-07T15:54:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T15:54:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/7155bd4b0e53419885801dcc25bb3281 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281 2024-11-07T15:54:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281, entries=150, sequenceid=243, filesize=11.9 K 2024-11-07T15:54:02,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,481 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 7dbbcb54c8468d54b4ae60ec53261963 in 137ms, sequenceid=243, compaction requested=true 2024-11-07T15:54:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-07T15:54:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-07T15:54:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-07T15:54:02,486 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 754 msec 2024-11-07T15:54:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,492 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 763 msec 2024-11-07T15:54:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:02,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,552 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:02,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:54:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754ef4fdbc50d417b84433fe1e0067920_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:02,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742019_1195 (size=24758) 2024-11-07T15:54:02,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994902637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994902640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994902753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994902753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-07T15:54:02,864 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:02,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:02,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-07T15:54:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:02,867 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:02,868 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:02,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:02,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994902963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994902964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:02,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:02,985 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:02,989 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110754ef4fdbc50d417b84433fe1e0067920_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754ef4fdbc50d417b84433fe1e0067920_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:02,990 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/d99d59e3d065499eaff159a2742d9020, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:02,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/d99d59e3d065499eaff159a2742d9020 is 175, key is test_row_0/A:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:02,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742018_1194 (size=74395) 2024-11-07T15:54:03,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994903273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:03,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994903273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:03,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/d99d59e3d065499eaff159a2742d9020 2024-11-07T15:54:03,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d85a07e31b9d4bbba4ba2d98c4f8a846 is 50, key is test_row_0/B:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:03,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742020_1196 (size=12151) 2024-11-07T15:54:03,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:03,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:03,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994903783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:03,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:03,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994903793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:03,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:03,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d85a07e31b9d4bbba4ba2d98c4f8a846 2024-11-07T15:54:03,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/5383a68613b247b7bf93825912d7944f is 50, key is test_row_0/C:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:03,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742021_1197 (size=12151) 2024-11-07T15:54:03,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:03,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:03,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:03,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:04,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35978 deadline: 1730994904182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,185 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36002 deadline: 1730994904193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,195 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35950 deadline: 1730994904203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,206 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 7dbbcb54c8468d54b4ae60ec53261963 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/5383a68613b247b7bf93825912d7944f 2024-11-07T15:54:04,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/d99d59e3d065499eaff159a2742d9020 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020 2024-11-07T15:54:04,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020, entries=400, sequenceid=254, filesize=72.7 K 2024-11-07T15:54:04,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/d85a07e31b9d4bbba4ba2d98c4f8a846 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846 2024-11-07T15:54:04,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:04,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846, entries=150, sequenceid=254, filesize=11.9 K 2024-11-07T15:54:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:04,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/5383a68613b247b7bf93825912d7944f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f 2024-11-07T15:54:04,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f, entries=150, sequenceid=254, filesize=11.9 K 2024-11-07T15:54:04,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7dbbcb54c8468d54b4ae60ec53261963 in 1705ms, sequenceid=254, compaction requested=true 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:04,257 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:04,257 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:54:04,259 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,259 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,259 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/775fa127d57d46189d0b3642791b0dba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=45.6 K 2024-11-07T15:54:04,259 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=172.7 K 2024-11-07T15:54:04,259 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020] 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 775fa127d57d46189d0b3642791b0dba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839874 2024-11-07T15:54:04,259 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cd92e3929234f63b34cf493e11f8724, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839874 2024-11-07T15:54:04,260 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 89ecb2d0e3c3460789626b8871ddc10d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730994840526 2024-11-07T15:54:04,260 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a61e4ec9be248e98a7948b963af526f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730994840526 2024-11-07T15:54:04,260 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1dba93e05c45aa9a8379487f9ba3d9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730994841801 2024-11-07T15:54:04,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bcbc12ced104e84ba8432f04dff7894, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730994841801 2024-11-07T15:54:04,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d85a07e31b9d4bbba4ba2d98c4f8a846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842541 2024-11-07T15:54:04,261 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting d99d59e3d065499eaff159a2742d9020, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842519 2024-11-07T15:54:04,271 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:04,275 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:04,275 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/124f83a67aef4f509aa3cc5fd0dac2f0 is 50, key is test_row_0/B:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:04,279 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110758f7439ec1204c1c900c5aa9e20a8295_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:04,283 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110758f7439ec1204c1c900c5aa9e20a8295_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:04,283 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110758f7439ec1204c1c900c5aa9e20a8295_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:04,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742023_1199 (size=4469) 2024-11-07T15:54:04,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742022_1198 (size=12731) 2024-11-07T15:54:04,310 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/124f83a67aef4f509aa3cc5fd0dac2f0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/124f83a67aef4f509aa3cc5fd0dac2f0 2024-11-07T15:54:04,317 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 124f83a67aef4f509aa3cc5fd0dac2f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:04,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:04,317 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=12, startTime=1730994844257; duration=0sec 2024-11-07T15:54:04,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:04,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:54:04,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:04,319 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:04,319 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:54:04,319 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,319 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/34664d4e814a44e990bae7c0ecbc2fba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=45.6 K 2024-11-07T15:54:04,319 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 34664d4e814a44e990bae7c0ecbc2fba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1730994839874 2024-11-07T15:54:04,320 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d6ce0eb3cdcd428c8d753acabab25f74, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1730994840526 2024-11-07T15:54:04,321 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7155bd4b0e53419885801dcc25bb3281, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1730994841801 2024-11-07T15:54:04,321 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5383a68613b247b7bf93825912d7944f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842541 2024-11-07T15:54:04,330 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:04,331 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/61658c23dbb0480c89dc15d8a04f334e is 50, key is test_row_0/C:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:04,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742024_1200 (size=12731) 2024-11-07T15:54:04,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-07T15:54:04,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:04,402 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:54:04,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b3e4da4a585c4c1caf0885606d05918b_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994842638/Put/seqid=0 2024-11-07T15:54:04,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742025_1201 (size=12454) 2024-11-07T15:54:04,700 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#165 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:04,701 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/6ac2f6cb2b6946bca0bfbe92cffbb15e is 175, key is test_row_0/A:col10/1730994842549/Put/seqid=0 2024-11-07T15:54:04,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742026_1202 (size=31685) 2024-11-07T15:54:04,747 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/61658c23dbb0480c89dc15d8a04f334e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61658c23dbb0480c89dc15d8a04f334e 2024-11-07T15:54:04,753 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 61658c23dbb0480c89dc15d8a04f334e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:04,753 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:04,753 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=12, startTime=1730994844257; duration=0sec 2024-11-07T15:54:04,753 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:04,753 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:54:04,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:04,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:04,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:04,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994904816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,823 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b3e4da4a585c4c1caf0885606d05918b_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3e4da4a585c4c1caf0885606d05918b_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e876daaa31884191ba38e37910184eff, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:04,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e876daaa31884191ba38e37910184eff is 175, key is test_row_0/A:col10/1730994842638/Put/seqid=0 2024-11-07T15:54:04,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994904822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742027_1203 (size=31255) 2024-11-07T15:54:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994904932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994904933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:04,938 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:05,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:05,111 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/6ac2f6cb2b6946bca0bfbe92cffbb15e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e 2024-11-07T15:54:05,116 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 6ac2f6cb2b6946bca0bfbe92cffbb15e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:05,117 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:05,117 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=12, startTime=1730994844257; duration=0sec 2024-11-07T15:54:05,117 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:05,117 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:54:05,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994905143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:05,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994905143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:05,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e876daaa31884191ba38e37910184eff 2024-11-07T15:54:05,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5603b5ae0f014d0295abaefb33174331 is 50, key is test_row_0/B:col10/1730994842638/Put/seqid=0 2024-11-07T15:54:05,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742028_1204 (size=12301) 2024-11-07T15:54:05,246 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5603b5ae0f014d0295abaefb33174331 2024-11-07T15:54:05,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1339f01d0bf443e5ac264df635bd8acd is 50, key is test_row_0/C:col10/1730994842638/Put/seqid=0 2024-11-07T15:54:05,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742029_1205 (size=12301) 2024-11-07T15:54:05,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994905453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:05,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994905453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:05,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:05,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1339f01d0bf443e5ac264df635bd8acd 2024-11-07T15:54:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/e876daaa31884191ba38e37910184eff as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff 2024-11-07T15:54:05,672 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff, entries=150, sequenceid=280, filesize=30.5 K 2024-11-07T15:54:05,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5603b5ae0f014d0295abaefb33174331 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331 2024-11-07T15:54:05,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T15:54:05,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1339f01d0bf443e5ac264df635bd8acd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd 2024-11-07T15:54:05,682 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T15:54:05,684 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7dbbcb54c8468d54b4ae60ec53261963 in 1282ms, sequenceid=280, compaction requested=false 2024-11-07T15:54:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:05,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-07T15:54:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-07T15:54:05,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-07T15:54:05,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8180 sec 2024-11-07T15:54:05,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.8240 sec 2024-11-07T15:54:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:05,964 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:05,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:54:05,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110747eaa60ecc2f4486bf6d113070f56a3d_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742030_1206 (size=14994) 2024-11-07T15:54:06,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994906016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994906018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994906123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994906123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994906333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994906333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,383 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:06,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110747eaa60ecc2f4486bf6d113070f56a3d_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747eaa60ecc2f4486bf6d113070f56a3d_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:06,390 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/5f6b73de11c14dfbbfa47f0f9f282402, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:06,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/5f6b73de11c14dfbbfa47f0f9f282402 is 175, key is test_row_0/A:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:06,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742031_1207 (size=39949) 2024-11-07T15:54:06,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994906643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994906643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:06,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:06,803 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/5f6b73de11c14dfbbfa47f0f9f282402 2024-11-07T15:54:06,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/fde1d5b1709f4bee8665c42f1facae1c is 50, key is test_row_0/B:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742032_1208 (size=12301) 2024-11-07T15:54:06,817 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/fde1d5b1709f4bee8665c42f1facae1c 2024-11-07T15:54:06,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1a46839d517f4d38befc150d9e62ce82 is 50, key is test_row_0/C:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:06,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742033_1209 (size=12301) 2024-11-07T15:54:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-07T15:54:07,024 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:07,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-07T15:54:07,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:07,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:07,027 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:07,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:07,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994907153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:07,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994907154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:07,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:07,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:07,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:07,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:54:07,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:07,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:07,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:07,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1a46839d517f4d38befc150d9e62ce82 2024-11-07T15:54:07,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/5f6b73de11c14dfbbfa47f0f9f282402 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402 2024-11-07T15:54:07,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402, entries=200, sequenceid=295, filesize=39.0 K 2024-11-07T15:54:07,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/fde1d5b1709f4bee8665c42f1facae1c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c 2024-11-07T15:54:07,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c, entries=150, sequenceid=295, filesize=12.0 K 2024-11-07T15:54:07,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1a46839d517f4d38befc150d9e62ce82 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82 2024-11-07T15:54:07,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82, entries=150, sequenceid=295, filesize=12.0 K 2024-11-07T15:54:07,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7dbbcb54c8468d54b4ae60ec53261963 in 1287ms, sequenceid=295, compaction requested=true 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:07,252 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:07,252 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:07,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:07,253 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:07,253 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:07,253 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:54:07,253 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:54:07,253 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,253 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,253 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=100.5 K 2024-11-07T15:54:07,253 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,253 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/124f83a67aef4f509aa3cc5fd0dac2f0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=36.5 K 2024-11-07T15:54:07,253 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402] 2024-11-07T15:54:07,254 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 124f83a67aef4f509aa3cc5fd0dac2f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842541 2024-11-07T15:54:07,254 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ac2f6cb2b6946bca0bfbe92cffbb15e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842541 2024-11-07T15:54:07,254 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5603b5ae0f014d0295abaefb33174331, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994842613 2024-11-07T15:54:07,255 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e876daaa31884191ba38e37910184eff, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994842613 2024-11-07T15:54:07,255 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fde1d5b1709f4bee8665c42f1facae1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:07,255 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f6b73de11c14dfbbfa47f0f9f282402, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:07,263 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#174 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:07,263 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/3d18c8baa7194cf493bc0bd73700ba63 is 50, key is test_row_0/B:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:07,264 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:07,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742034_1210 (size=12983) 2024-11-07T15:54:07,268 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411074e4eaff5d8fb4f648dab0072f2345eec_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:07,269 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411074e4eaff5d8fb4f648dab0072f2345eec_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:07,269 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074e4eaff5d8fb4f648dab0072f2345eec_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:07,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742035_1211 (size=4469) 2024-11-07T15:54:07,279 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#175 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:07,279 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54b803c6548b4355a2c4295c341d1cc4 is 175, key is test_row_0/A:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:07,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742036_1212 (size=31937) 2024-11-07T15:54:07,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-07T15:54:07,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,333 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:54:07,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110788a3b52061574e10a528b0a37638553d_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994846016/Put/seqid=0 2024-11-07T15:54:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:07,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742037_1213 (size=12454) 2024-11-07T15:54:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:07,674 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/3d18c8baa7194cf493bc0bd73700ba63 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/3d18c8baa7194cf493bc0bd73700ba63 2024-11-07T15:54:07,680 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 3d18c8baa7194cf493bc0bd73700ba63(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:07,680 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:07,680 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=13, startTime=1730994847252; duration=0sec 2024-11-07T15:54:07,680 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:07,680 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:54:07,680 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:07,681 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:07,682 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:54:07,682 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:07,682 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61658c23dbb0480c89dc15d8a04f334e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=36.5 K 2024-11-07T15:54:07,682 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 61658c23dbb0480c89dc15d8a04f334e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1730994842541 2024-11-07T15:54:07,683 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1339f01d0bf443e5ac264df635bd8acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994842613 2024-11-07T15:54:07,684 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a46839d517f4d38befc150d9e62ce82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:07,693 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/54b803c6548b4355a2c4295c341d1cc4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4 2024-11-07T15:54:07,695 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:07,695 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/f7ff1262fd6f459eb99c35eb5e4391d6 is 50, key is test_row_0/C:col10/1730994845964/Put/seqid=0 2024-11-07T15:54:07,700 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 54b803c6548b4355a2c4295c341d1cc4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:07,700 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:07,700 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=13, startTime=1730994847252; duration=0sec 2024-11-07T15:54:07,700 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:07,700 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:54:07,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742038_1214 (size=12983) 2024-11-07T15:54:07,714 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/f7ff1262fd6f459eb99c35eb5e4391d6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f7ff1262fd6f459eb99c35eb5e4391d6 2024-11-07T15:54:07,721 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into f7ff1262fd6f459eb99c35eb5e4391d6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:07,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:07,722 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=13, startTime=1730994847252; duration=0sec 2024-11-07T15:54:07,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:07,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:54:07,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:07,753 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110788a3b52061574e10a528b0a37638553d_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788a3b52061574e10a528b0a37638553d_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/16eedefdcf314d3c896927bf351619f3, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:07,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/16eedefdcf314d3c896927bf351619f3 is 175, key is test_row_0/A:col10/1730994846016/Put/seqid=0 2024-11-07T15:54:07,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742039_1215 (size=31255) 2024-11-07T15:54:07,760 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/16eedefdcf314d3c896927bf351619f3 2024-11-07T15:54:07,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9e7b203fc12f4cfd8e73300ae1165e13 is 50, key is test_row_0/B:col10/1730994846016/Put/seqid=0 2024-11-07T15:54:07,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742040_1216 (size=12301) 2024-11-07T15:54:08,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:08,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:08,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:08,176 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9e7b203fc12f4cfd8e73300ae1165e13 2024-11-07T15:54:08,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994908195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/2c4b80b514c249da95754dbad59df378 is 50, key is test_row_0/C:col10/1730994846016/Put/seqid=0 2024-11-07T15:54:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,204 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994908205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742041_1217 (size=12301) 2024-11-07T15:54:08,221 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/2c4b80b514c249da95754dbad59df378 2024-11-07T15:54:08,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/16eedefdcf314d3c896927bf351619f3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3 2024-11-07T15:54:08,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3, entries=150, sequenceid=318, filesize=30.5 K 2024-11-07T15:54:08,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9e7b203fc12f4cfd8e73300ae1165e13 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13 2024-11-07T15:54:08,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13, entries=150, sequenceid=318, filesize=12.0 K 2024-11-07T15:54:08,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/2c4b80b514c249da95754dbad59df378 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378 2024-11-07T15:54:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378, entries=150, sequenceid=318, filesize=12.0 K 2024-11-07T15:54:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,254 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7dbbcb54c8468d54b4ae60ec53261963 in 921ms, sequenceid=318, compaction requested=false 2024-11-07T15:54:08,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:08,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:08,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-07T15:54:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-07T15:54:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-07T15:54:08,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2300 sec 2024-11-07T15:54:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.2360 sec 2024-11-07T15:54:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:08,321 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:08,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:54:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ca8c325dde44288aee0f6747145e862_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742042_1218 (size=14994) 2024-11-07T15:54:08,357 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:08,363 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ca8c325dde44288aee0f6747145e862_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca8c325dde44288aee0f6747145e862_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:08,365 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/b8cc5112849a44528556404df34b7700, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:08,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/b8cc5112849a44528556404df34b7700 is 175, key is test_row_0/A:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742043_1219 (size=39949) 2024-11-07T15:54:08,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/b8cc5112849a44528556404df34b7700 2024-11-07T15:54:08,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994908387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994908389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/b2373de45cdb4a208c9628dab42329f6 is 50, key is test_row_0/B:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742044_1220 (size=12301) 2024-11-07T15:54:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994908503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994908503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994908713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994908713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:08,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/b2373de45cdb4a208c9628dab42329f6 2024-11-07T15:54:08,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/3aaec4070a1e42caba91a21ea403dfa8 is 50, key is test_row_0/C:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742045_1221 (size=12301) 2024-11-07T15:54:08,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/3aaec4070a1e42caba91a21ea403dfa8 2024-11-07T15:54:08,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/b8cc5112849a44528556404df34b7700 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700 2024-11-07T15:54:08,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700, entries=200, sequenceid=336, filesize=39.0 K 2024-11-07T15:54:08,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/b2373de45cdb4a208c9628dab42329f6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6 2024-11-07T15:54:08,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6, entries=150, sequenceid=336, filesize=12.0 K 2024-11-07T15:54:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/3aaec4070a1e42caba91a21ea403dfa8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8 2024-11-07T15:54:08,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8, entries=150, sequenceid=336, filesize=12.0 K 2024-11-07T15:54:08,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7dbbcb54c8468d54b4ae60ec53261963 in 563ms, sequenceid=336, compaction requested=true 2024-11-07T15:54:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:08,885 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:08,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:08,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:08,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:54:08,886 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:08,886 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:08,886 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:54:08,887 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:08,887 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=100.7 K 2024-11-07T15:54:08,887 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:08,887 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700] 2024-11-07T15:54:08,892 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54b803c6548b4355a2c4295c341d1cc4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:08,892 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16eedefdcf314d3c896927bf351619f3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730994846002 2024-11-07T15:54:08,893 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:08,893 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:54:08,893 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:08,893 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8cc5112849a44528556404df34b7700, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:08,893 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/3d18c8baa7194cf493bc0bd73700ba63, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=36.7 K 2024-11-07T15:54:08,893 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d18c8baa7194cf493bc0bd73700ba63, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:08,894 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e7b203fc12f4cfd8e73300ae1165e13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730994846002 2024-11-07T15:54:08,895 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b2373de45cdb4a208c9628dab42329f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:08,917 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:08,922 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:08,923 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/dac4e3c1732b4e2fa7fad09bd246bd9d is 50, key is test_row_0/B:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,925 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110776ab098d5c704641941e32a209f1e878_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:08,926 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110776ab098d5c704641941e32a209f1e878_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:08,926 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110776ab098d5c704641941e32a209f1e878_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:08,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742046_1222 (size=13085) 2024-11-07T15:54:08,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742047_1223 (size=4469) 2024-11-07T15:54:08,939 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#183 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:08,940 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/f89e6bff14d6495fa0bf49c7ed0445fc is 175, key is test_row_0/A:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:08,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742048_1224 (size=32039) 2024-11-07T15:54:08,958 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/f89e6bff14d6495fa0bf49c7ed0445fc as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc 2024-11-07T15:54:08,967 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into f89e6bff14d6495fa0bf49c7ed0445fc(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:08,967 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:08,967 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=13, startTime=1730994848885; duration=0sec 2024-11-07T15:54:08,967 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:08,967 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:54:08,967 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:08,969 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:08,969 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:54:08,969 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:08,969 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f7ff1262fd6f459eb99c35eb5e4391d6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=36.7 K 2024-11-07T15:54:08,971 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7ff1262fd6f459eb99c35eb5e4391d6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994844815 2024-11-07T15:54:08,972 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4b80b514c249da95754dbad59df378, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1730994846002 2024-11-07T15:54:08,972 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aaec4070a1e42caba91a21ea403dfa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:08,990 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#185 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:08,991 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/e2df8d5108564d9b822d994f10fef329 is 50, key is test_row_0/C:col10/1730994848319/Put/seqid=0 2024-11-07T15:54:09,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742049_1225 (size=13085) 2024-11-07T15:54:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:09,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:54:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:09,028 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/e2df8d5108564d9b822d994f10fef329 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e2df8d5108564d9b822d994f10fef329 2024-11-07T15:54:09,042 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into e2df8d5108564d9b822d994f10fef329(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:09,042 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:09,042 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=13, startTime=1730994848886; duration=0sec 2024-11-07T15:54:09,042 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:09,042 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:54:09,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071d09b149cfb54c389dd05a9c2946c511_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994849024/Put/seqid=0 2024-11-07T15:54:09,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994909058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994909059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742050_1226 (size=17534) 2024-11-07T15:54:09,070 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:09,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071d09b149cfb54c389dd05a9c2946c511_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d09b149cfb54c389dd05a9c2946c511_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:09,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/fe3d9c1f162a4caa83fc694b70cc5c6d, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:09,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/fe3d9c1f162a4caa83fc694b70cc5c6d is 175, key is test_row_0/A:col10/1730994849024/Put/seqid=0 2024-11-07T15:54:09,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742051_1227 (size=48639) 2024-11-07T15:54:09,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/fe3d9c1f162a4caa83fc694b70cc5c6d 2024-11-07T15:54:09,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/249911e18ab44649bfc6fc8066dc589c is 50, key is test_row_0/B:col10/1730994849024/Put/seqid=0 2024-11-07T15:54:09,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742052_1228 (size=12301) 2024-11-07T15:54:09,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/249911e18ab44649bfc6fc8066dc589c 2024-11-07T15:54:09,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/39a901ec5f6b4c3caaa508180a6ba712 is 50, key is test_row_0/C:col10/1730994849024/Put/seqid=0 2024-11-07T15:54:09,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742053_1229 (size=12301) 2024-11-07T15:54:09,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-07T15:54:09,173 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:09,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:09,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-07T15:54:09,176 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:09,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:09,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:09,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:09,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994909173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994909173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:09,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:54:09,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:09,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,341 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/dac4e3c1732b4e2fa7fad09bd246bd9d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/dac4e3c1732b4e2fa7fad09bd246bd9d 2024-11-07T15:54:09,349 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into dac4e3c1732b4e2fa7fad09bd246bd9d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:09,349 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:09,349 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=13, startTime=1730994848885; duration=0sec 2024-11-07T15:54:09,349 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:09,349 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:54:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994909383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994909384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:54:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:09,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:09,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/39a901ec5f6b4c3caaa508180a6ba712 2024-11-07T15:54:09,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/fe3d9c1f162a4caa83fc694b70cc5c6d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d 2024-11-07T15:54:09,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d, entries=250, sequenceid=360, filesize=47.5 K 2024-11-07T15:54:09,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/249911e18ab44649bfc6fc8066dc589c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c 2024-11-07T15:54:09,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T15:54:09,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/39a901ec5f6b4c3caaa508180a6ba712 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712 2024-11-07T15:54:09,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712, entries=150, sequenceid=360, filesize=12.0 K 2024-11-07T15:54:09,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7dbbcb54c8468d54b4ae60ec53261963 in 557ms, sequenceid=360, compaction requested=false 2024-11-07T15:54:09,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:09,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-07T15:54:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:09,639 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:54:09,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110751bbe9c31bf64ebf8c3168a151016bdd_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994849056/Put/seqid=0 2024-11-07T15:54:09,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742054_1230 (size=12454) 2024-11-07T15:54:09,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:09,661 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110751bbe9c31bf64ebf8c3168a151016bdd_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751bbe9c31bf64ebf8c3168a151016bdd_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d3fe01dd8994dd9a2c88f235a2cfe4b, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:09,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d3fe01dd8994dd9a2c88f235a2cfe4b is 175, key is test_row_0/A:col10/1730994849056/Put/seqid=0 2024-11-07T15:54:09,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742055_1231 (size=31255) 2024-11-07T15:54:09,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994909745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994909746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:09,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994909853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994909853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:09,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994910063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994910063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,066 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,069 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d3fe01dd8994dd9a2c88f235a2cfe4b 2024-11-07T15:54:10,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9c2ace8bd3924cc2bd7884247d70eabd is 50, key is test_row_0/B:col10/1730994849056/Put/seqid=0 2024-11-07T15:54:10,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742056_1232 (size=12301) 2024-11-07T15:54:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:10,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994910373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994910373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9c2ace8bd3924cc2bd7884247d70eabd 2024-11-07T15:54:10,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/66b279f63f0c453d80b44c056404b9cc is 50, key is test_row_0/C:col10/1730994849056/Put/seqid=0 2024-11-07T15:54:10,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742057_1233 (size=12301) 2024-11-07T15:54:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994910883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994910883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:10,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:10,900 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/66b279f63f0c453d80b44c056404b9cc 2024-11-07T15:54:10,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/3d3fe01dd8994dd9a2c88f235a2cfe4b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b 2024-11-07T15:54:10,912 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b, entries=150, sequenceid=375, filesize=30.5 K 2024-11-07T15:54:10,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9c2ace8bd3924cc2bd7884247d70eabd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd 2024-11-07T15:54:10,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd, entries=150, sequenceid=375, filesize=12.0 K 2024-11-07T15:54:10,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/66b279f63f0c453d80b44c056404b9cc as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc 2024-11-07T15:54:10,924 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc, entries=150, sequenceid=375, filesize=12.0 K 2024-11-07T15:54:10,926 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7dbbcb54c8468d54b4ae60ec53261963 in 1288ms, sequenceid=375, compaction requested=true 2024-11-07T15:54:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:10,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-07T15:54:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-07T15:54:10,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-07T15:54:10,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7510 sec 2024-11-07T15:54:10,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.7580 sec 2024-11-07T15:54:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-07T15:54:11,323 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:11,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:11,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-07T15:54:11,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:11,325 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:11,326 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:11,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:11,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-07T15:54:11,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:11,479 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:54:11,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107869510fe52c34f5989c2888be3478c1b_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994849745/Put/seqid=0 2024-11-07T15:54:11,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742058_1234 (size=12454) 2024-11-07T15:54:11,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:11,500 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107869510fe52c34f5989c2888be3478c1b_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107869510fe52c34f5989c2888be3478c1b_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:11,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/9cf267b9f84c463ba8d75b1a9e5c82ca, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:11,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/9cf267b9f84c463ba8d75b1a9e5c82ca is 175, key is test_row_0/A:col10/1730994849745/Put/seqid=0 2024-11-07T15:54:11,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742059_1235 (size=31255) 2024-11-07T15:54:11,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:11,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. as already flushing 2024-11-07T15:54:11,916 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/9cf267b9f84c463ba8d75b1a9e5c82ca 2024-11-07T15:54:11,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994911916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:11,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:11,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:11,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994911920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:11,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/c701f2b2340547caa510e253830af42d is 50, key is test_row_0/B:col10/1730994849745/Put/seqid=0 2024-11-07T15:54:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:11,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:11,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742060_1236 (size=12301) 2024-11-07T15:54:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:12,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994912033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994912033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994912243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994912243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/c701f2b2340547caa510e253830af42d 2024-11-07T15:54:12,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/573769e2a22843908d7ae3a37f727f8e is 50, key is test_row_0/C:col10/1730994849745/Put/seqid=0 2024-11-07T15:54:12,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742061_1237 (size=12301) 2024-11-07T15:54:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:12,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994912553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994912553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:12,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:12,747 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/573769e2a22843908d7ae3a37f727f8e 2024-11-07T15:54:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/9cf267b9f84c463ba8d75b1a9e5c82ca as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca 2024-11-07T15:54:12,757 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca, entries=150, sequenceid=398, filesize=30.5 K 2024-11-07T15:54:12,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/c701f2b2340547caa510e253830af42d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d 2024-11-07T15:54:12,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d, entries=150, sequenceid=398, filesize=12.0 K 2024-11-07T15:54:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/573769e2a22843908d7ae3a37f727f8e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e 2024-11-07T15:54:12,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e, entries=150, sequenceid=398, filesize=12.0 K 2024-11-07T15:54:12,770 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7dbbcb54c8468d54b4ae60ec53261963 in 1291ms, sequenceid=398, compaction requested=true 2024-11-07T15:54:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:12,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-07T15:54:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-07T15:54:12,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-07T15:54:12,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-07T15:54:12,775 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.4510 sec 2024-11-07T15:54:13,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:13,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:54:13,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070495a5b8a42e48e1aba138bf06af7675_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:13,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742062_1238 (size=17534) 2024-11-07T15:54:13,080 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:13,085 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070495a5b8a42e48e1aba138bf06af7675_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070495a5b8a42e48e1aba138bf06af7675_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:13,086 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/aae0481957c3434bbbf712dd261539bf, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:13,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/aae0481957c3434bbbf712dd261539bf is 175, key is test_row_0/A:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:13,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742063_1239 (size=48639) 2024-11-07T15:54:13,100 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/aae0481957c3434bbbf712dd261539bf 2024-11-07T15:54:13,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9d052b77381348d1828a9b481b54f8d1 is 50, key is test_row_0/B:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:13,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742064_1240 (size=12301) 2024-11-07T15:54:13,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994913111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994913114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994913223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994913223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,231 INFO [Thread-666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-666. 2024-11-07T15:54:13,232 DEBUG [Thread-666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:13,232 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,232 DEBUG [Thread-666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,232 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:13,234 INFO [Thread-660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-660. 2024-11-07T15:54:13,234 DEBUG [Thread-660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:13,234 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,234 DEBUG [Thread-660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,234 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:13,235 INFO [Thread-664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-664. 2024-11-07T15:54:13,235 DEBUG [Thread-664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:13,235 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,235 DEBUG [Thread-664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,235 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:13,236 INFO [Thread-662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-662. 2024-11-07T15:54:13,236 DEBUG [Thread-662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:13,236 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,236 DEBUG [Thread-662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:13,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994913433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994913433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,434 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-07T15:54:13,474 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:13,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9d052b77381348d1828a9b481b54f8d1 2024-11-07T15:54:13,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/4a8cff0b050841d0ab086e5b08aa4afe is 50, key is test_row_0/C:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:13,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742065_1241 (size=12301) 2024-11-07T15:54:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35992 deadline: 1730994913743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35942 deadline: 1730994913743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:13,927 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/4a8cff0b050841d0ab086e5b08aa4afe 2024-11-07T15:54:13,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/aae0481957c3434bbbf712dd261539bf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf 2024-11-07T15:54:13,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf, entries=250, sequenceid=413, filesize=47.5 K 2024-11-07T15:54:13,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9d052b77381348d1828a9b481b54f8d1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1 2024-11-07T15:54:13,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T15:54:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/4a8cff0b050841d0ab086e5b08aa4afe as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe 2024-11-07T15:54:13,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe, entries=150, sequenceid=413, filesize=12.0 K 2024-11-07T15:54:13,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7dbbcb54c8468d54b4ae60ec53261963 in 882ms, sequenceid=413, compaction requested=true 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:13,946 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7dbbcb54c8468d54b4ae60ec53261963:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:13,946 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:54:13,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 191827 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/B is initiating minor compaction (all files) 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/A is initiating minor compaction (all files) 2024-11-07T15:54:13,948 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/A in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:13,948 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/B in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:13,948 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=187.3 K 2024-11-07T15:54:13,948 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/dac4e3c1732b4e2fa7fad09bd246bd9d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=60.8 K 2024-11-07T15:54:13,948 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf] 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting dac4e3c1732b4e2fa7fad09bd246bd9d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:13,948 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting f89e6bff14d6495fa0bf49c7ed0445fc, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 249911e18ab44649bfc6fc8066dc589c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730994848368 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe3d9c1f162a4caa83fc694b70cc5c6d, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730994848333 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2ace8bd3924cc2bd7884247d70eabd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1730994849051 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d3fe01dd8994dd9a2c88f235a2cfe4b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1730994849051 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c701f2b2340547caa510e253830af42d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730994849736 2024-11-07T15:54:13,949 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cf267b9f84c463ba8d75b1a9e5c82ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730994849736 2024-11-07T15:54:13,950 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d052b77381348d1828a9b481b54f8d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730994851918 2024-11-07T15:54:13,950 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting aae0481957c3434bbbf712dd261539bf, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730994851908 2024-11-07T15:54:13,974 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:13,976 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#B#compaction#198 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:13,976 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9236cc3c10234368b1421e813a89194a is 50, key is test_row_0/B:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:13,977 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107c5de507499564318a7d143d6fd267e4b_7dbbcb54c8468d54b4ae60ec53261963 store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:13,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742066_1242 (size=13255) 2024-11-07T15:54:13,998 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107c5de507499564318a7d143d6fd267e4b_7dbbcb54c8468d54b4ae60ec53261963, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:13,998 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c5de507499564318a7d143d6fd267e4b_7dbbcb54c8468d54b4ae60ec53261963 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:14,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742067_1243 (size=4469) 2024-11-07T15:54:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:14,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:14,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:54:14,215 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-07T15:54:14,215 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:14,215 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,215 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,215 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f27934b7c4774cd0806d251282ab8e1e_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994853111/Put/seqid=0 2024-11-07T15:54:14,224 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-07T15:54:14,224 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:14,224 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,225 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,225 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-11-07T15:54:14,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,225 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:14,225 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,225 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742068_1244 (size=12454) 2024-11-07T15:54:14,255 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-07T15:54:14,255 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:14,255 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,255 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,255 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-11-07T15:54:14,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,255 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:14,255 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,255 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3662 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3482 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1539 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4617 rows 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1530 2024-11-07T15:54:14,256 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4587 rows 2024-11-07T15:54:14,256 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:54:14,256 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:54:14,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:14,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:14,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:54:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:14,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:14,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994854260"}]},"ts":"1730994854260"} 2024-11-07T15:54:14,261 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:54:14,261 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:54:14,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:54:14,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, UNASSIGN}] 2024-11-07T15:54:14,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, UNASSIGN 2024-11-07T15:54:14,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:14,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, UNASSIGN because future has completed 2024-11-07T15:54:14,266 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:54:14,266 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:14,385 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/9236cc3c10234368b1421e813a89194a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9236cc3c10234368b1421e813a89194a 2024-11-07T15:54:14,390 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/B of 7dbbcb54c8468d54b4ae60ec53261963 into 9236cc3c10234368b1421e813a89194a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:14,390 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:14,390 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/B, priority=11, startTime=1730994853946; duration=0sec 2024-11-07T15:54:14,390 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:14,390 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:B 2024-11-07T15:54:14,390 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-07T15:54:14,391 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-07T15:54:14,391 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 7dbbcb54c8468d54b4ae60ec53261963/C is initiating minor compaction (all files) 2024-11-07T15:54:14,391 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7dbbcb54c8468d54b4ae60ec53261963/C in TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:14,391 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e2df8d5108564d9b822d994f10fef329, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp, totalSize=60.8 K 2024-11-07T15:54:14,392 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e2df8d5108564d9b822d994f10fef329, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1730994848194 2024-11-07T15:54:14,392 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 39a901ec5f6b4c3caaa508180a6ba712, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1730994848368 2024-11-07T15:54:14,393 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b279f63f0c453d80b44c056404b9cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1730994849051 2024-11-07T15:54:14,393 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 573769e2a22843908d7ae3a37f727f8e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1730994849736 2024-11-07T15:54:14,393 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a8cff0b050841d0ab086e5b08aa4afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1730994851918 2024-11-07T15:54:14,402 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#C#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:14,402 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1aaa9d8c361a4d0a8d8eb9df8f0a70f0 is 50, key is test_row_0/C:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:14,403 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7dbbcb54c8468d54b4ae60ec53261963#A#compaction#199 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:14,404 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/95437a3f570445f9988d2fea2239226b is 175, key is test_row_0/A:col10/1730994853063/Put/seqid=0 2024-11-07T15:54:14,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742069_1245 (size=13255) 2024-11-07T15:54:14,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742070_1246 (size=32209) 2024-11-07T15:54:14,418 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:14,418 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:54:14,418 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 7dbbcb54c8468d54b4ae60ec53261963, disabling compactions & flushes 2024-11-07T15:54:14,418 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:14,627 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:14,631 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f27934b7c4774cd0806d251282ab8e1e_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f27934b7c4774cd0806d251282ab8e1e_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:14,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/701258d0037f4b32a84eb37e16b3721e, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:14,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/701258d0037f4b32a84eb37e16b3721e is 175, key is test_row_0/A:col10/1730994853111/Put/seqid=0 2024-11-07T15:54:14,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742071_1247 (size=31255) 2024-11-07T15:54:14,812 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/1aaa9d8c361a4d0a8d8eb9df8f0a70f0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1aaa9d8c361a4d0a8d8eb9df8f0a70f0 2024-11-07T15:54:14,813 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/95437a3f570445f9988d2fea2239226b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/95437a3f570445f9988d2fea2239226b 2024-11-07T15:54:14,817 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/C of 7dbbcb54c8468d54b4ae60ec53261963 into 1aaa9d8c361a4d0a8d8eb9df8f0a70f0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:14,817 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/C, priority=11, startTime=1730994853946; duration=0sec 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:C 2024-11-07T15:54:14,817 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 7dbbcb54c8468d54b4ae60ec53261963/A of 7dbbcb54c8468d54b4ae60ec53261963 into 95437a3f570445f9988d2fea2239226b(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:14,817 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:14,817 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., storeName=7dbbcb54c8468d54b4ae60ec53261963/A, priority=11, startTime=1730994853946; duration=0sec 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:14,817 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7dbbcb54c8468d54b4ae60ec53261963:A 2024-11-07T15:54:14,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:14,925 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:54:15,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=435, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/701258d0037f4b32a84eb37e16b3721e 2024-11-07T15:54:15,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/2b68f611920a4bc1916bfdfa3dd455a9 is 50, key is test_row_0/B:col10/1730994853111/Put/seqid=0 2024-11-07T15:54:15,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742072_1248 (size=12301) 2024-11-07T15:54:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:15,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/2b68f611920a4bc1916bfdfa3dd455a9 2024-11-07T15:54:15,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/02bf3cea33d34a9ea1e5c1114cefb1e1 is 50, key is test_row_0/C:col10/1730994853111/Put/seqid=0 2024-11-07T15:54:15,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742073_1249 (size=12301) 2024-11-07T15:54:15,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/02bf3cea33d34a9ea1e5c1114cefb1e1 2024-11-07T15:54:15,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/701258d0037f4b32a84eb37e16b3721e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/701258d0037f4b32a84eb37e16b3721e 2024-11-07T15:54:15,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/701258d0037f4b32a84eb37e16b3721e, entries=150, sequenceid=435, filesize=30.5 K 2024-11-07T15:54:15,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/2b68f611920a4bc1916bfdfa3dd455a9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/2b68f611920a4bc1916bfdfa3dd455a9 2024-11-07T15:54:15,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/2b68f611920a4bc1916bfdfa3dd455a9, entries=150, sequenceid=435, filesize=12.0 K 2024-11-07T15:54:15,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/02bf3cea33d34a9ea1e5c1114cefb1e1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/02bf3cea33d34a9ea1e5c1114cefb1e1 2024-11-07T15:54:15,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/02bf3cea33d34a9ea1e5c1114cefb1e1, entries=150, sequenceid=435, filesize=12.0 K 2024-11-07T15:54:15,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=26.84 KB/27480 for 7dbbcb54c8468d54b4ae60ec53261963 in 1664ms, sequenceid=435, compaction requested=false 2024-11-07T15:54:15,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7dbbcb54c8468d54b4ae60ec53261963: 2024-11-07T15:54:15,878 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:15,878 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:15,878 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. after waiting 0 ms 2024-11-07T15:54:15,878 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:15,878 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 7dbbcb54c8468d54b4ae60ec53261963 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T15:54:15,884 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f8c4ec5bd8e47928ed173ea89f16a86_7dbbcb54c8468d54b4ae60ec53261963 is 50, key is test_row_0/A:col10/1730994854254/Put/seqid=0 2024-11-07T15:54:15,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742074_1250 (size=12454) 2024-11-07T15:54:16,288 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:16,292 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070f8c4ec5bd8e47928ed173ea89f16a86_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8c4ec5bd8e47928ed173ea89f16a86_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:16,292 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/afc166cc53c446fea4633d5b58c845fb, store: [table=TestAcidGuarantees family=A region=7dbbcb54c8468d54b4ae60ec53261963] 2024-11-07T15:54:16,293 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/afc166cc53c446fea4633d5b58c845fb is 175, key is test_row_0/A:col10/1730994854254/Put/seqid=0 2024-11-07T15:54:16,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742075_1251 (size=31255) 2024-11-07T15:54:16,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:16,697 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/afc166cc53c446fea4633d5b58c845fb 2024-11-07T15:54:16,704 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5e394a6b54484e8ca91b0a8e6a831d0f is 50, key is test_row_0/B:col10/1730994854254/Put/seqid=0 2024-11-07T15:54:16,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742076_1252 (size=12301) 2024-11-07T15:54:17,108 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5e394a6b54484e8ca91b0a8e6a831d0f 2024-11-07T15:54:17,115 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/dafc781ca6064198b3958239d030b0a6 is 50, key is test_row_0/C:col10/1730994854254/Put/seqid=0 2024-11-07T15:54:17,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742077_1253 (size=12301) 2024-11-07T15:54:17,519 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/dafc781ca6064198b3958239d030b0a6 2024-11-07T15:54:17,523 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/A/afc166cc53c446fea4633d5b58c845fb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/afc166cc53c446fea4633d5b58c845fb 2024-11-07T15:54:17,527 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/afc166cc53c446fea4633d5b58c845fb, entries=150, sequenceid=445, filesize=30.5 K 2024-11-07T15:54:17,527 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/B/5e394a6b54484e8ca91b0a8e6a831d0f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5e394a6b54484e8ca91b0a8e6a831d0f 2024-11-07T15:54:17,531 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5e394a6b54484e8ca91b0a8e6a831d0f, entries=150, sequenceid=445, filesize=12.0 K 2024-11-07T15:54:17,532 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/.tmp/C/dafc781ca6064198b3958239d030b0a6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/dafc781ca6064198b3958239d030b0a6 2024-11-07T15:54:17,535 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/dafc781ca6064198b3958239d030b0a6, entries=150, sequenceid=445, filesize=12.0 K 2024-11-07T15:54:17,536 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7dbbcb54c8468d54b4ae60ec53261963 in 1658ms, sequenceid=445, compaction requested=true 2024-11-07T15:54:17,536 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf] to archive 2024-11-07T15:54:17,537 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:17,539 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54213e53c8ad443fa1136b8fa66a9d68 2024-11-07T15:54:17,540 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a9c97cb8b1394c8f970711ce209c86d4 2024-11-07T15:54:17,541 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/938a613eb3b44a36a376996871730773 2024-11-07T15:54:17,542 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/496b04648e8c48539f522fbe0f6a42ba 2024-11-07T15:54:17,543 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/2ffca5a2e8ec4278986e3c2b4ffca667 2024-11-07T15:54:17,544 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/a3e693bc936a42028d9bdec55b149f07 2024-11-07T15:54:17,545 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e06947690de047f492372c8fc5337362 2024-11-07T15:54:17,546 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/26815ebf6d6e4eb4873e5465d6529f42 2024-11-07T15:54:17,547 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/c5c9172a428846a7a0f62f2a555993f3 2024-11-07T15:54:17,548 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d9d644860be4930a2b2477d4c436334 2024-11-07T15:54:17,549 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eac2636413a14ff78fbd91bba07a7592 2024-11-07T15:54:17,550 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/eb85e0da23a847719d806b74e7d61a5e 2024-11-07T15:54:17,551 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/7b37e3bd37e145498506a39a1d42d549 2024-11-07T15:54:17,552 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6cd92e3929234f63b34cf493e11f8724 2024-11-07T15:54:17,553 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a61e4ec9be248e98a7948b963af526f 2024-11-07T15:54:17,554 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/1a1dba93e05c45aa9a8379487f9ba3d9 2024-11-07T15:54:17,555 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/d99d59e3d065499eaff159a2742d9020 2024-11-07T15:54:17,557 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/6ac2f6cb2b6946bca0bfbe92cffbb15e 2024-11-07T15:54:17,558 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/e876daaa31884191ba38e37910184eff 2024-11-07T15:54:17,559 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/5f6b73de11c14dfbbfa47f0f9f282402 2024-11-07T15:54:17,560 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/54b803c6548b4355a2c4295c341d1cc4 2024-11-07T15:54:17,561 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/16eedefdcf314d3c896927bf351619f3 2024-11-07T15:54:17,562 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/b8cc5112849a44528556404df34b7700 2024-11-07T15:54:17,563 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/f89e6bff14d6495fa0bf49c7ed0445fc 2024-11-07T15:54:17,564 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/fe3d9c1f162a4caa83fc694b70cc5c6d 2024-11-07T15:54:17,565 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/3d3fe01dd8994dd9a2c88f235a2cfe4b 2024-11-07T15:54:17,566 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/9cf267b9f84c463ba8d75b1a9e5c82ca 2024-11-07T15:54:17,568 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/aae0481957c3434bbbf712dd261539bf 2024-11-07T15:54:17,569 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/09e34fadfa8c4a0c9870fc230952328a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/891533516ff340a1ac09e5613b8d177a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/86eb074732b241a7aec780fe6e243538, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/775fa127d57d46189d0b3642791b0dba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/124f83a67aef4f509aa3cc5fd0dac2f0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/3d18c8baa7194cf493bc0bd73700ba63, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/dac4e3c1732b4e2fa7fad09bd246bd9d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1] to archive 2024-11-07T15:54:17,570 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:17,572 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d3c8096100244b05a5723b01bc0d7b36 2024-11-07T15:54:17,573 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/13271af13ddb40eebd3ecfd33b65b570 2024-11-07T15:54:17,574 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/cd01448a88994613964fbe0951f99faf 2024-11-07T15:54:17,575 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/09e34fadfa8c4a0c9870fc230952328a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/09e34fadfa8c4a0c9870fc230952328a 2024-11-07T15:54:17,576 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d8438e60d0bd4ae8bcd89da8080d4846 2024-11-07T15:54:17,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/85b0c4682eba458fa82cb99c1f779b6e 2024-11-07T15:54:17,577 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/891533516ff340a1ac09e5613b8d177a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/891533516ff340a1ac09e5613b8d177a 2024-11-07T15:54:17,578 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/411abd592b1347d7b4279720b00fb75d 2024-11-07T15:54:17,579 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/4175c0ffc58d44e198c0c77799f15b3b 2024-11-07T15:54:17,581 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/86eb074732b241a7aec780fe6e243538 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/86eb074732b241a7aec780fe6e243538 2024-11-07T15:54:17,582 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/87a96e8ef1de4d8aa15bfad017c858f4 2024-11-07T15:54:17,583 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/af771974de1948a5a23e929b5fb695bd 2024-11-07T15:54:17,584 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/775fa127d57d46189d0b3642791b0dba to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/775fa127d57d46189d0b3642791b0dba 2024-11-07T15:54:17,585 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/954f2c3e2ee74724a3828366e61ea537 2024-11-07T15:54:17,586 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/89ecb2d0e3c3460789626b8871ddc10d 2024-11-07T15:54:17,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/6bcbc12ced104e84ba8432f04dff7894 2024-11-07T15:54:17,587 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/124f83a67aef4f509aa3cc5fd0dac2f0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/124f83a67aef4f509aa3cc5fd0dac2f0 2024-11-07T15:54:17,588 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/d85a07e31b9d4bbba4ba2d98c4f8a846 2024-11-07T15:54:17,589 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5603b5ae0f014d0295abaefb33174331 2024-11-07T15:54:17,590 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/3d18c8baa7194cf493bc0bd73700ba63 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/3d18c8baa7194cf493bc0bd73700ba63 2024-11-07T15:54:17,591 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/fde1d5b1709f4bee8665c42f1facae1c 2024-11-07T15:54:17,592 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9e7b203fc12f4cfd8e73300ae1165e13 2024-11-07T15:54:17,593 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/dac4e3c1732b4e2fa7fad09bd246bd9d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/dac4e3c1732b4e2fa7fad09bd246bd9d 2024-11-07T15:54:17,594 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/b2373de45cdb4a208c9628dab42329f6 2024-11-07T15:54:17,595 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/249911e18ab44649bfc6fc8066dc589c 2024-11-07T15:54:17,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9c2ace8bd3924cc2bd7884247d70eabd 2024-11-07T15:54:17,596 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/c701f2b2340547caa510e253830af42d 2024-11-07T15:54:17,597 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9d052b77381348d1828a9b481b54f8d1 2024-11-07T15:54:17,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/70e0e6ade38a442cbdb2119153a5955a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/8889dea86d0f40ffa5a57c98f60ae02d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/47a0f7cf14264af39a02bf24e6c7298f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/34664d4e814a44e990bae7c0ecbc2fba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61658c23dbb0480c89dc15d8a04f334e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f7ff1262fd6f459eb99c35eb5e4391d6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e2df8d5108564d9b822d994f10fef329, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe] to archive 2024-11-07T15:54:17,599 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:17,601 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/748e2486e37c48ef87c4bc33425d7ccd 2024-11-07T15:54:17,602 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/49a27671dd7a45e6b9e3020499748aa3 2024-11-07T15:54:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e3eba4d66d7f41c88db65f73a3e13e1a 2024-11-07T15:54:17,603 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/70e0e6ade38a442cbdb2119153a5955a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/70e0e6ade38a442cbdb2119153a5955a 2024-11-07T15:54:17,604 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61b0c1da538d47418dc6a14677ff3220 2024-11-07T15:54:17,605 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7c45295d95614b70b0a1df0ad349cca0 2024-11-07T15:54:17,606 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/8889dea86d0f40ffa5a57c98f60ae02d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/8889dea86d0f40ffa5a57c98f60ae02d 2024-11-07T15:54:17,607 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f8b2b6c184bd49419678d863c8bb4fe9 2024-11-07T15:54:17,608 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ff403ae07c824df4a5140a08ac4170fb 2024-11-07T15:54:17,609 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/47a0f7cf14264af39a02bf24e6c7298f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/47a0f7cf14264af39a02bf24e6c7298f 2024-11-07T15:54:17,610 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/28cbc5b9bde648ba9dd07786c93f9e38 2024-11-07T15:54:17,611 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7bd586eb3e7a431db230c9ea62aad683 2024-11-07T15:54:17,612 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/34664d4e814a44e990bae7c0ecbc2fba to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/34664d4e814a44e990bae7c0ecbc2fba 2024-11-07T15:54:17,613 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/ffbf28c0383f44a9b68be98afc0d0f40 2024-11-07T15:54:17,614 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/d6ce0eb3cdcd428c8d753acabab25f74 2024-11-07T15:54:17,615 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/7155bd4b0e53419885801dcc25bb3281 2024-11-07T15:54:17,616 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61658c23dbb0480c89dc15d8a04f334e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/61658c23dbb0480c89dc15d8a04f334e 2024-11-07T15:54:17,617 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/5383a68613b247b7bf93825912d7944f 2024-11-07T15:54:17,618 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1339f01d0bf443e5ac264df635bd8acd 2024-11-07T15:54:17,619 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f7ff1262fd6f459eb99c35eb5e4391d6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/f7ff1262fd6f459eb99c35eb5e4391d6 2024-11-07T15:54:17,620 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1a46839d517f4d38befc150d9e62ce82 2024-11-07T15:54:17,621 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/2c4b80b514c249da95754dbad59df378 2024-11-07T15:54:17,622 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e2df8d5108564d9b822d994f10fef329 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/e2df8d5108564d9b822d994f10fef329 2024-11-07T15:54:17,622 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/3aaec4070a1e42caba91a21ea403dfa8 2024-11-07T15:54:17,623 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/39a901ec5f6b4c3caaa508180a6ba712 2024-11-07T15:54:17,624 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/66b279f63f0c453d80b44c056404b9cc 2024-11-07T15:54:17,625 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/573769e2a22843908d7ae3a37f727f8e 2024-11-07T15:54:17,626 DEBUG [StoreCloser-TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/4a8cff0b050841d0ab086e5b08aa4afe 2024-11-07T15:54:17,630 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits/448.seqid, newMaxSeqId=448, maxSeqId=4 2024-11-07T15:54:17,631 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. 2024-11-07T15:54:17,631 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 7dbbcb54c8468d54b4ae60ec53261963: Waiting for close lock at 1730994854418Running coprocessor pre-close hooks at 1730994854418Disabling compacts and flushes for region at 1730994854418Disabling writes for close at 1730994855878 (+1460 ms)Obtaining lock to block concurrent updates at 1730994855878Preparing flush snapshotting stores in 7dbbcb54c8468d54b4ae60ec53261963 at 1730994855878Finished memstore snapshotting TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1730994855878Flushing stores of TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963. at 1730994855879 (+1 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/A: creating writer at 1730994855879Flushing mob file 7dbbcb54c8468d54b4ae60ec53261963/A: appending metadata at 1730994855884 (+5 ms)Flushing mob file 7dbbcb54c8468d54b4ae60ec53261963/A: closing flushed file at 1730994855884Flushing 7dbbcb54c8468d54b4ae60ec53261963/A: appending metadata at 1730994856292 (+408 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/A: closing flushed file at 1730994856292Flushing 7dbbcb54c8468d54b4ae60ec53261963/B: creating writer at 1730994856701 (+409 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/B: appending metadata at 1730994856704 (+3 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/B: closing flushed file at 1730994856704Flushing 7dbbcb54c8468d54b4ae60ec53261963/C: creating writer at 1730994857112 (+408 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/C: appending metadata at 1730994857114 (+2 ms)Flushing 7dbbcb54c8468d54b4ae60ec53261963/C: closing flushed file at 1730994857114Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6799f22b: reopening flushed file at 1730994857523 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f3b061: reopening flushed file at 1730994857527 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@150ccaf6: reopening flushed file at 1730994857531 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 7dbbcb54c8468d54b4ae60ec53261963 in 1658ms, sequenceid=445, compaction requested=true at 1730994857536 (+5 ms)Writing region close event to WAL at 1730994857627 (+91 ms)Running coprocessor post-close hooks at 1730994857631 (+4 ms)Closed at 1730994857631 2024-11-07T15:54:17,633 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:17,634 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7dbbcb54c8468d54b4ae60ec53261963, regionState=CLOSED 2024-11-07T15:54:17,635 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:17,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T15:54:17,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 7dbbcb54c8468d54b4ae60ec53261963, server=c2682f91c1e2,42309,1730994796599 in 3.3700 sec 2024-11-07T15:54:17,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-07T15:54:17,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7dbbcb54c8468d54b4ae60ec53261963, UNASSIGN in 3.3740 sec 2024-11-07T15:54:17,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T15:54:17,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.3780 sec 2024-11-07T15:54:17,642 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994857642"}]},"ts":"1730994857642"} 2024-11-07T15:54:17,643 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:54:17,643 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:54:17,645 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.3870 sec 2024-11-07T15:54:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-07T15:54:18,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:18,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:54:18,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,426 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T15:54:18,427 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,429 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,432 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits] 2024-11-07T15:54:18,435 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/701258d0037f4b32a84eb37e16b3721e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/701258d0037f4b32a84eb37e16b3721e 2024-11-07T15:54:18,436 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/95437a3f570445f9988d2fea2239226b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/95437a3f570445f9988d2fea2239226b 2024-11-07T15:54:18,437 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/afc166cc53c446fea4633d5b58c845fb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/A/afc166cc53c446fea4633d5b58c845fb 2024-11-07T15:54:18,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/2b68f611920a4bc1916bfdfa3dd455a9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/2b68f611920a4bc1916bfdfa3dd455a9 2024-11-07T15:54:18,441 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5e394a6b54484e8ca91b0a8e6a831d0f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/5e394a6b54484e8ca91b0a8e6a831d0f 2024-11-07T15:54:18,443 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9236cc3c10234368b1421e813a89194a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/B/9236cc3c10234368b1421e813a89194a 2024-11-07T15:54:18,445 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/02bf3cea33d34a9ea1e5c1114cefb1e1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/02bf3cea33d34a9ea1e5c1114cefb1e1 2024-11-07T15:54:18,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1aaa9d8c361a4d0a8d8eb9df8f0a70f0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/1aaa9d8c361a4d0a8d8eb9df8f0a70f0 2024-11-07T15:54:18,447 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/dafc781ca6064198b3958239d030b0a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/C/dafc781ca6064198b3958239d030b0a6 2024-11-07T15:54:18,450 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits/448.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963/recovered.edits/448.seqid 2024-11-07T15:54:18,451 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,451 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:54:18,451 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:54:18,452 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:54:18,456 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070495a5b8a42e48e1aba138bf06af7675_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070495a5b8a42e48e1aba138bf06af7675_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,458 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070869c3c9a0224a6082ecd86f5d9de098_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070869c3c9a0224a6082ecd86f5d9de098_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8c4ec5bd8e47928ed173ea89f16a86_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070f8c4ec5bd8e47928ed173ea89f16a86_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107109f1a5e8d174e738e2fe947b0b2d2a9_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107109f1a5e8d174e738e2fe947b0b2d2a9_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107156f1b76748843cca58154363f3e63e3_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107156f1b76748843cca58154363f3e63e3_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d09b149cfb54c389dd05a9c2946c511_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071d09b149cfb54c389dd05a9c2946c511_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,465 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073364da2bc03f44b08d0469d6bb92bfa6_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411073364da2bc03f44b08d0469d6bb92bfa6_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,466 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747eaa60ecc2f4486bf6d113070f56a3d_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110747eaa60ecc2f4486bf6d113070f56a3d_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,467 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751bbe9c31bf64ebf8c3168a151016bdd_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110751bbe9c31bf64ebf8c3168a151016bdd_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,469 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754ef4fdbc50d417b84433fe1e0067920_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110754ef4fdbc50d417b84433fe1e0067920_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,470 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757acf7982e184c66b4b5759459115b0a_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110757acf7982e184c66b4b5759459115b0a_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,471 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fab5cb8de6a4560b9768dffd0127259_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075fab5cb8de6a4560b9768dffd0127259_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,473 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765db9de9d529498f8ff9842c9c76dd41_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110765db9de9d529498f8ff9842c9c76dd41_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,474 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772f9f6d355da45348c45b774d8182ae4_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110772f9f6d355da45348c45b774d8182ae4_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107869510fe52c34f5989c2888be3478c1b_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107869510fe52c34f5989c2888be3478c1b_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,478 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786c2baa9e5b842a09f5cfd991a911d05_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110786c2baa9e5b842a09f5cfd991a911d05_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,479 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788a3b52061574e10a528b0a37638553d_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110788a3b52061574e10a528b0a37638553d_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,481 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca8c325dde44288aee0f6747145e862_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ca8c325dde44288aee0f6747145e862_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,482 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3e4da4a585c4c1caf0885606d05918b_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107b3e4da4a585c4c1caf0885606d05918b_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,484 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107caeeba9f646341aba0ab87d3e17a4e65_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107caeeba9f646341aba0ab87d3e17a4e65_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,485 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dda4fca6a6bd4cd8a3e54f84db4f13c6_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107dda4fca6a6bd4cd8a3e54f84db4f13c6_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,486 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e8642cb059f54656a4532568fc7a1bc3_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e8642cb059f54656a4532568fc7a1bc3_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,487 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f27934b7c4774cd0806d251282ab8e1e_7dbbcb54c8468d54b4ae60ec53261963 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f27934b7c4774cd0806d251282ab8e1e_7dbbcb54c8468d54b4ae60ec53261963 2024-11-07T15:54:18,488 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:54:18,490 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,493 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:54:18,495 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:54:18,497 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,497 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:54:18,497 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994858497"}]},"ts":"9223372036854775807"} 2024-11-07T15:54:18,499 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:54:18,499 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7dbbcb54c8468d54b4ae60ec53261963, NAME => 'TestAcidGuarantees,,1730994830086.7dbbcb54c8468d54b4ae60ec53261963.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:54:18,499 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:54:18,499 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994858499"}]},"ts":"9223372036854775807"} 2024-11-07T15:54:18,501 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:54:18,502 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-07T15:54:18,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-07T15:54:18,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:54:18,533 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:18,546 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=241 (was 236) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_816498657_22 at /127.0.0.1:48724 [Waiting for operation #650] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_816498657_22 at /127.0.0.1:48776 [Waiting for operation #565] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-9504082_22 at /127.0.0.1:56496 [Waiting for operation #724] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-9504082_22 at /127.0.0.1:48614 [Waiting for operation #696] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 212) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5651 (was 5781) 2024-11-07T15:54:18,556 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=5656 2024-11-07T15:54:18,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:54:18,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:54:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:18,560 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:54:18,561 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:18,561 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-07T15:54:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:18,562 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:54:18,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742078_1254 (size=959) 2024-11-07T15:54:18,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:18,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:54:18,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742079_1255 (size=53) 2024-11-07T15:54:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:19,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:54:19,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c94656b3cdcc591544c8dc0ab185fdfc, disabling compactions & flushes 2024-11-07T15:54:19,376 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. after waiting 0 ms 2024-11-07T15:54:19,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,376 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c94656b3cdcc591544c8dc0ab185fdfc: Waiting for close lock at 1730994859375Disabling compacts and flushes for region at 1730994859375Disabling writes for close at 1730994859376 (+1 ms)Writing region close event to WAL at 1730994859376Closed at 1730994859376 2024-11-07T15:54:19,377 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:54:19,377 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994859377"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994859377"}]},"ts":"1730994859377"} 2024-11-07T15:54:19,379 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:54:19,379 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:54:19,380 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994859380"}]},"ts":"1730994859380"} 2024-11-07T15:54:19,381 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:54:19,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, ASSIGN}] 2024-11-07T15:54:19,382 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, ASSIGN 2024-11-07T15:54:19,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:54:19,533 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c94656b3cdcc591544c8dc0ab185fdfc, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:19,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, ASSIGN because future has completed 2024-11-07T15:54:19,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:19,690 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,690 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:54:19,691 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,691 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:54:19,691 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,691 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,692 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,693 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c94656b3cdcc591544c8dc0ab185fdfc columnFamilyName A 2024-11-07T15:54:19,694 DEBUG [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:19,694 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(327): Store=c94656b3cdcc591544c8dc0ab185fdfc/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:19,694 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,695 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c94656b3cdcc591544c8dc0ab185fdfc columnFamilyName B 2024-11-07T15:54:19,695 DEBUG [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:19,696 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(327): Store=c94656b3cdcc591544c8dc0ab185fdfc/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:19,696 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,697 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c94656b3cdcc591544c8dc0ab185fdfc columnFamilyName C 2024-11-07T15:54:19,697 DEBUG [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:19,697 INFO [StoreOpener-c94656b3cdcc591544c8dc0ab185fdfc-1 {}] regionserver.HStore(327): Store=c94656b3cdcc591544c8dc0ab185fdfc/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:19,697 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,698 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,698 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,699 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,699 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,699 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:54:19,700 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,702 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:54:19,703 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened c94656b3cdcc591544c8dc0ab185fdfc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65384014, jitterRate=-0.02570226788520813}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:54:19,703 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:19,703 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for c94656b3cdcc591544c8dc0ab185fdfc: Running coprocessor pre-open hook at 1730994859691Writing region info on filesystem at 1730994859691Initializing all the Stores at 1730994859692 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994859692Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994859692Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994859692Cleaning up temporary data from old regions at 1730994859699 (+7 ms)Running coprocessor post-open hooks at 1730994859703 (+4 ms)Region opened successfully at 1730994859703 2024-11-07T15:54:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:19,704 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., pid=65, masterSystemTime=1730994859687 2024-11-07T15:54:19,706 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,706 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:19,706 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c94656b3cdcc591544c8dc0ab185fdfc, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:19,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:19,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-07T15:54:19,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 in 173 msec 2024-11-07T15:54:19,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-07T15:54:19,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, ASSIGN in 328 msec 2024-11-07T15:54:19,712 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:54:19,712 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994859712"}]},"ts":"1730994859712"} 2024-11-07T15:54:19,713 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:54:19,714 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:54:19,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-07T15:54:20,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-07T15:54:20,713 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:20,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213da200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59168, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d414ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,718 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,721 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:54:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:54:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46cf66dd 2024-11-07T15:54:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:54:20,722 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:54:20,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,724 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b593e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,726 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f831df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,727 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,727 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,727 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,728 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa6a85e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,729 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd5a943, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,730 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@493fe5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfe720f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59380f26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,734 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dfa38b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,735 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,735 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef080b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@377ee933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49bee9e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caef57a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,742 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e5b962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,743 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad77c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,744 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,744 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61672d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b59678, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,747 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5445bde2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74299b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:20,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:20,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:20,749 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd865f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:20,750 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:20,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40096380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:20,752 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [Thread-1199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [Thread-1197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [Thread-1195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,753 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,753 DEBUG [Thread-1180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,753 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,753 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:20,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:20,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-07T15:54:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,757 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,759 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:20,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,760 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:54:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:20,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:20,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:54:20,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50814cfbd9514adbb39e4fa483f793f2 is 50, key is test_row_0/A:col10/1730994860764/Put/seqid=0 2024-11-07T15:54:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994920806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994920809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994920812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994920814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994920814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742080_1256 (size=12001) 2024-11-07T15:54:20,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50814cfbd9514adbb39e4fa483f793f2 2024-11-07T15:54:20,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:20,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87a5c3067fd043bb95e0e23d6702060d is 50, key is test_row_0/B:col10/1730994860764/Put/seqid=0 2024-11-07T15:54:20,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742081_1257 (size=12001) 2024-11-07T15:54:20,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:20,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:20,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:20,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:20,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:20,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994920924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994920924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994920924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994920933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994920933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:20,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:21,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994921133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,136 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994921136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994921136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994921143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994921143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87a5c3067fd043bb95e0e23d6702060d 2024-11-07T15:54:21,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/3566854f37464e7d96b4c17849c2873c is 50, key is test_row_0/C:col10/1730994860764/Put/seqid=0 2024-11-07T15:54:21,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742082_1258 (size=12001) 2024-11-07T15:54:21,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/3566854f37464e7d96b4c17849c2873c 2024-11-07T15:54:21,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50814cfbd9514adbb39e4fa483f793f2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2 2024-11-07T15:54:21,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:54:21,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87a5c3067fd043bb95e0e23d6702060d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d 2024-11-07T15:54:21,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:54:21,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/3566854f37464e7d96b4c17849c2873c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c 2024-11-07T15:54:21,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:54:21,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c94656b3cdcc591544c8dc0ab185fdfc in 639ms, sequenceid=14, compaction requested=false 2024-11-07T15:54:21,407 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:54:21,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:21,447 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:21,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:54:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:21,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/14000e0bbead456ea8dad6a400b845e6 is 50, key is test_row_0/A:col10/1730994861444/Put/seqid=0 2024-11-07T15:54:21,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742083_1259 (size=12001) 2024-11-07T15:54:21,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/14000e0bbead456ea8dad6a400b845e6 2024-11-07T15:54:21,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/cb0909c44db646239a7d203f25ed2f40 is 50, key is test_row_0/B:col10/1730994861444/Put/seqid=0 2024-11-07T15:54:21,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742084_1260 (size=12001) 2024-11-07T15:54:21,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994921465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994921470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994921526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994921527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994921527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994921633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994921633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994921633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994921634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994921634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,638 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994921843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994921843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994921843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994921843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994921844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:21,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:21,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:21,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/cb0909c44db646239a7d203f25ed2f40 2024-11-07T15:54:21,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/db574cd3c94f4418b1c34adcb5f4b629 is 50, key is test_row_0/C:col10/1730994861444/Put/seqid=0 2024-11-07T15:54:21,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742085_1261 (size=12001) 2024-11-07T15:54:21,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:21,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:21,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:22,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:22,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994922153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994922153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994922153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994922154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994922154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:22,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:22,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:22,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/db574cd3c94f4418b1c34adcb5f4b629 2024-11-07T15:54:22,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/14000e0bbead456ea8dad6a400b845e6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6 2024-11-07T15:54:22,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:54:22,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/cb0909c44db646239a7d203f25ed2f40 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40 2024-11-07T15:54:22,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:54:22,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/db574cd3c94f4418b1c34adcb5f4b629 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629 2024-11-07T15:54:22,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629, entries=150, sequenceid=38, filesize=11.7 K 2024-11-07T15:54:22,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for c94656b3cdcc591544c8dc0ab185fdfc in 885ms, sequenceid=38, compaction requested=false 2024-11-07T15:54:22,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:22,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-07T15:54:22,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:22,458 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:54:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/38a0123278564425af0d6457078f8655 is 50, key is test_row_0/A:col10/1730994861478/Put/seqid=0 2024-11-07T15:54:22,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742086_1262 (size=12001) 2024-11-07T15:54:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:22,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:22,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994922683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994922686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994922687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994922688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994922688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994922793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994922803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994922803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994922804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994922804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:22,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:22,869 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/38a0123278564425af0d6457078f8655 2024-11-07T15:54:22,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/d9aa871935c54a238ec0b770823e7eb4 is 50, key is test_row_0/B:col10/1730994861478/Put/seqid=0 2024-11-07T15:54:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742087_1263 (size=12001) 2024-11-07T15:54:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:23,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994923003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994923012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994923013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994923013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994923013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,152 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:54:23,283 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/d9aa871935c54a238ec0b770823e7eb4 2024-11-07T15:54:23,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7cb8ba761b0c4923a309c82b08966806 is 50, key is test_row_0/C:col10/1730994861478/Put/seqid=0 2024-11-07T15:54:23,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742088_1264 (size=12001) 2024-11-07T15:54:23,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994923313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994923323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994923323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994923323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994923323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7cb8ba761b0c4923a309c82b08966806 2024-11-07T15:54:23,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/38a0123278564425af0d6457078f8655 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655 2024-11-07T15:54:23,705 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T15:54:23,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/d9aa871935c54a238ec0b770823e7eb4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4 2024-11-07T15:54:23,710 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T15:54:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7cb8ba761b0c4923a309c82b08966806 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806 2024-11-07T15:54:23,715 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806, entries=150, sequenceid=51, filesize=11.7 K 2024-11-07T15:54:23,715 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c94656b3cdcc591544c8dc0ab185fdfc in 1257ms, sequenceid=51, compaction requested=true 2024-11-07T15:54:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:23,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-07T15:54:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-07T15:54:23,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-07T15:54:23,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9580 sec 2024-11-07T15:54:23,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.9650 sec 2024-11-07T15:54:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:23,825 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:23,825 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:54:23,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/8184c0638c0a4b4fa38e2fd18fa655f9 is 50, key is test_row_0/A:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:23,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742089_1265 (size=14341) 2024-11-07T15:54:23,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994923838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994923839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994923842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994923843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994923844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994923953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994923953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994923953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994923953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994923953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:23,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994924163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994924163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994924163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994924164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994924164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/8184c0638c0a4b4fa38e2fd18fa655f9 2024-11-07T15:54:24,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/61e7ab4bd3ca435d8740165eeadb44fa is 50, key is test_row_0/B:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:24,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742090_1266 (size=12001) 2024-11-07T15:54:24,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994924473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994924473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994924473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994924473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994924473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:24,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:24,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/61e7ab4bd3ca435d8740165eeadb44fa 2024-11-07T15:54:24,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/f7ae640585cc4a5abeb6d1887b98eee6 is 50, key is test_row_0/C:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:24,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742091_1267 (size=12001) 2024-11-07T15:54:24,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/f7ae640585cc4a5abeb6d1887b98eee6 2024-11-07T15:54:24,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/8184c0638c0a4b4fa38e2fd18fa655f9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9 2024-11-07T15:54:24,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9, entries=200, sequenceid=75, filesize=14.0 K 2024-11-07T15:54:24,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/61e7ab4bd3ca435d8740165eeadb44fa as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa 2024-11-07T15:54:24,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T15:54:24,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/f7ae640585cc4a5abeb6d1887b98eee6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6 2024-11-07T15:54:24,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6, entries=150, sequenceid=75, filesize=11.7 K 2024-11-07T15:54:24,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c94656b3cdcc591544c8dc0ab185fdfc in 857ms, sequenceid=75, compaction requested=true 2024-11-07T15:54:24,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:24,683 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:24,683 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:24,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:24,684 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:24,684 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:24,684 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/A is initiating minor compaction (all files) 2024-11-07T15:54:24,684 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/B is initiating minor compaction (all files) 2024-11-07T15:54:24,684 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/A in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:24,684 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/B in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:24,684 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=49.2 K 2024-11-07T15:54:24,684 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=46.9 K 2024-11-07T15:54:24,685 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50814cfbd9514adbb39e4fa483f793f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994860764 2024-11-07T15:54:24,685 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 87a5c3067fd043bb95e0e23d6702060d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994860764 2024-11-07T15:54:24,685 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0909c44db646239a7d203f25ed2f40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730994860803 2024-11-07T15:54:24,686 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14000e0bbead456ea8dad6a400b845e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730994860803 2024-11-07T15:54:24,686 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d9aa871935c54a238ec0b770823e7eb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994861458 2024-11-07T15:54:24,686 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a0123278564425af0d6457078f8655, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994861458 2024-11-07T15:54:24,686 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 61e7ab4bd3ca435d8740165eeadb44fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:24,687 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8184c0638c0a4b4fa38e2fd18fa655f9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:24,698 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:24,698 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/fa79e79128ac449e9e00ce61cc243189 is 50, key is test_row_0/A:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:24,699 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#B#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:24,699 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/8d9ca673145846b9adbc015414e14b13 is 50, key is test_row_0/B:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:24,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742092_1268 (size=12139) 2024-11-07T15:54:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742093_1269 (size=12139) 2024-11-07T15:54:24,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/8d9ca673145846b9adbc015414e14b13 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/8d9ca673145846b9adbc015414e14b13 2024-11-07T15:54:24,728 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/B of c94656b3cdcc591544c8dc0ab185fdfc into 8d9ca673145846b9adbc015414e14b13(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:24,728 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:24,728 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/B, priority=12, startTime=1730994864683; duration=0sec 2024-11-07T15:54:24,728 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:24,728 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:B 2024-11-07T15:54:24,728 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:24,730 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:24,730 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/C is initiating minor compaction (all files) 2024-11-07T15:54:24,730 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/C in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:24,730 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=46.9 K 2024-11-07T15:54:24,731 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3566854f37464e7d96b4c17849c2873c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994860764 2024-11-07T15:54:24,731 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting db574cd3c94f4418b1c34adcb5f4b629, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1730994860803 2024-11-07T15:54:24,732 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb8ba761b0c4923a309c82b08966806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1730994861458 2024-11-07T15:54:24,732 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting f7ae640585cc4a5abeb6d1887b98eee6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:24,741 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:24,742 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/d3b4c99fdea34afd997c02148532fd7c is 50, key is test_row_0/C:col10/1730994862679/Put/seqid=0 2024-11-07T15:54:24,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742094_1270 (size=12139) 2024-11-07T15:54:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-07T15:54:24,923 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:24,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-07T15:54:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:24,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:24,926 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:24,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:24,986 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:24,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:54:24,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/faa93816ce3546858c1167f9dde019bb is 50, key is test_row_0/A:col10/1730994864985/Put/seqid=0 2024-11-07T15:54:24,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742095_1271 (size=12001) 2024-11-07T15:54:24,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/faa93816ce3546858c1167f9dde019bb 2024-11-07T15:54:25,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/4763d2179ebe49a9930197a51001b74f is 50, key is test_row_0/B:col10/1730994864985/Put/seqid=0 2024-11-07T15:54:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994925010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994925011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994925013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994925014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994925015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742096_1272 (size=12001) 2024-11-07T15:54:25,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/4763d2179ebe49a9930197a51001b74f 2024-11-07T15:54:25,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a842dcc3b1244b9181843247bbe8d3e8 is 50, key is test_row_0/C:col10/1730994864985/Put/seqid=0 2024-11-07T15:54:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:25,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742097_1273 (size=12001) 2024-11-07T15:54:25,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a842dcc3b1244b9181843247bbe8d3e8 2024-11-07T15:54:25,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/faa93816ce3546858c1167f9dde019bb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb 2024-11-07T15:54:25,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T15:54:25,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/4763d2179ebe49a9930197a51001b74f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f 2024-11-07T15:54:25,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T15:54:25,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a842dcc3b1244b9181843247bbe8d3e8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8 2024-11-07T15:54:25,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8, entries=150, sequenceid=91, filesize=11.7 K 2024-11-07T15:54:25,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c94656b3cdcc591544c8dc0ab185fdfc in 67ms, sequenceid=91, compaction requested=false 2024-11-07T15:54:25,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:25,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-07T15:54:25,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:25,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:54:25,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/b811babe8e73487f9606e08a850d0490 is 50, key is test_row_0/A:col10/1730994865013/Put/seqid=0 2024-11-07T15:54:25,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742098_1274 (size=12001) 2024-11-07T15:54:25,121 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/fa79e79128ac449e9e00ce61cc243189 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/fa79e79128ac449e9e00ce61cc243189 2024-11-07T15:54:25,126 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/A of c94656b3cdcc591544c8dc0ab185fdfc into fa79e79128ac449e9e00ce61cc243189(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:25,126 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:25,126 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/A, priority=12, startTime=1730994864682; duration=0sec 2024-11-07T15:54:25,126 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:25,126 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:A 2024-11-07T15:54:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:25,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994925137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994925137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994925138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994925139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994925140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,154 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/d3b4c99fdea34afd997c02148532fd7c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/d3b4c99fdea34afd997c02148532fd7c 2024-11-07T15:54:25,159 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/C of c94656b3cdcc591544c8dc0ab185fdfc into d3b4c99fdea34afd997c02148532fd7c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:25,159 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:25,159 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/C, priority=12, startTime=1730994864683; duration=0sec 2024-11-07T15:54:25,159 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:25,159 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:C 2024-11-07T15:54:25,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:25,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994925253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994925253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994925253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994925253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994925253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994925463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994925463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,467 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994925463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994925464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994925464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/b811babe8e73487f9606e08a850d0490 2024-11-07T15:54:25,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 is 50, key is test_row_0/B:col10/1730994865013/Put/seqid=0 2024-11-07T15:54:25,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742099_1275 (size=12001) 2024-11-07T15:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994925773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994925773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994925774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994925774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994925774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:25,906 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 2024-11-07T15:54:25,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c1a2772f20da461095c85b83fee6ec1c is 50, key is test_row_0/C:col10/1730994865013/Put/seqid=0 2024-11-07T15:54:25,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742100_1276 (size=12001) 2024-11-07T15:54:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:26,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994926283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:26,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994926283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:26,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994926283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:26,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994926283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:26,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994926283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:26,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c1a2772f20da461095c85b83fee6ec1c 2024-11-07T15:54:26,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/b811babe8e73487f9606e08a850d0490 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490 2024-11-07T15:54:26,329 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:54:26,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 2024-11-07T15:54:26,335 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:54:26,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c1a2772f20da461095c85b83fee6ec1c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c 2024-11-07T15:54:26,340 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c, entries=150, sequenceid=112, filesize=11.7 K 2024-11-07T15:54:26,342 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for c94656b3cdcc591544c8dc0ab185fdfc in 1263ms, sequenceid=112, compaction requested=true 2024-11-07T15:54:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:26,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-07T15:54:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-07T15:54:26,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-07T15:54:26,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-07T15:54:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4220 sec 2024-11-07T15:54:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-07T15:54:27,083 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:27,083 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-07T15:54:27,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:54:27,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:27,087 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:27,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:54:27,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-07T15:54:27,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:27,240 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:54:27,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/458cfea18896468d954cb81180e69b29 is 50, key is test_row_0/A:col10/1730994865135/Put/seqid=0 2024-11-07T15:54:27,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742101_1277 (size=12001) 2024-11-07T15:54:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994927315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994927315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994927316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994927317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994927320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:54:27,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994927423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994927433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994927433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994927433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994927433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994927633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994927643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994927643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994927643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994927643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/458cfea18896468d954cb81180e69b29 2024-11-07T15:54:27,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/3b77544c190342bfb81886cf6609b34a is 50, key is test_row_0/B:col10/1730994865135/Put/seqid=0 2024-11-07T15:54:27,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742102_1278 (size=12001) 2024-11-07T15:54:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:54:27,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994927943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994927953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994927953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994927953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994927953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,066 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/3b77544c190342bfb81886cf6609b34a 2024-11-07T15:54:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/2db23bc474cf4626b8b8793a716fff74 is 50, key is test_row_0/C:col10/1730994865135/Put/seqid=0 2024-11-07T15:54:28,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742103_1279 (size=12001) 2024-11-07T15:54:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/2db23bc474cf4626b8b8793a716fff74 2024-11-07T15:54:28,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/458cfea18896468d954cb81180e69b29 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29 2024-11-07T15:54:28,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29, entries=150, sequenceid=129, filesize=11.7 K 2024-11-07T15:54:28,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/3b77544c190342bfb81886cf6609b34a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a 2024-11-07T15:54:28,092 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a, entries=150, sequenceid=129, filesize=11.7 K 2024-11-07T15:54:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/2db23bc474cf4626b8b8793a716fff74 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74 2024-11-07T15:54:28,098 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74, entries=150, sequenceid=129, filesize=11.7 K 2024-11-07T15:54:28,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c94656b3cdcc591544c8dc0ab185fdfc in 859ms, sequenceid=129, compaction requested=true 2024-11-07T15:54:28,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:28,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:28,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-07T15:54:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-07T15:54:28,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-07T15:54:28,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-07T15:54:28,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.0210 sec 2024-11-07T15:54:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-07T15:54:28,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:28,224 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-07T15:54:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:54:28,226 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:28,227 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:28,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:54:28,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-07T15:54:28,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:28,380 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:54:28,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/6dfc6d54db974ebb8b07df18506abcaa is 50, key is test_row_0/A:col10/1730994867314/Put/seqid=0 2024-11-07T15:54:28,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742104_1280 (size=12151) 2024-11-07T15:54:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:28,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994928476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994928485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994928504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994928504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994928504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:54:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994928613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994928613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994928613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994928614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994928614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,617 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,791 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/6dfc6d54db974ebb8b07df18506abcaa 2024-11-07T15:54:28,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/df757a1c3ea54c6c97c4927442d0191c is 50, key is test_row_0/B:col10/1730994867314/Put/seqid=0 2024-11-07T15:54:28,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742105_1281 (size=12151) 2024-11-07T15:54:28,803 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/df757a1c3ea54c6c97c4927442d0191c 2024-11-07T15:54:28,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/cf106d0d0c01409a8e2ce3f56a2d3d67 is 50, key is test_row_0/C:col10/1730994867314/Put/seqid=0 2024-11-07T15:54:28,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742106_1282 (size=12151) 2024-11-07T15:54:28,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994928823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994928823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994928824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994928824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,828 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994928824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:54:29,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994929133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994929133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994929134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994929134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994929134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/cf106d0d0c01409a8e2ce3f56a2d3d67 2024-11-07T15:54:29,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/6dfc6d54db974ebb8b07df18506abcaa as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa 2024-11-07T15:54:29,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa, entries=150, sequenceid=150, filesize=11.9 K 2024-11-07T15:54:29,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/df757a1c3ea54c6c97c4927442d0191c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c 2024-11-07T15:54:29,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c, entries=150, sequenceid=150, filesize=11.9 K 2024-11-07T15:54:29,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/cf106d0d0c01409a8e2ce3f56a2d3d67 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67 2024-11-07T15:54:29,242 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67, entries=150, sequenceid=150, filesize=11.9 K 2024-11-07T15:54:29,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=93.93 KB/96180 for c94656b3cdcc591544c8dc0ab185fdfc in 864ms, sequenceid=150, compaction requested=true 2024-11-07T15:54:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:29,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:29,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-07T15:54:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-07T15:54:29,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T15:54:29,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0180 sec 2024-11-07T15:54:29,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0230 sec 2024-11-07T15:54:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-07T15:54:29,364 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:29,364 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-07T15:54:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:54:29,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:29,367 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:29,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:54:29,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-07T15:54:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:29,519 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:54:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/78030a7cf39d4b659c24ae4e56836b81 is 50, key is test_row_0/A:col10/1730994868470/Put/seqid=0 2024-11-07T15:54:29,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742107_1283 (size=12151) 2024-11-07T15:54:29,530 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/78030a7cf39d4b659c24ae4e56836b81 2024-11-07T15:54:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/051e06a389e9497384994d055e8aa5bf is 50, key is test_row_0/B:col10/1730994868470/Put/seqid=0 2024-11-07T15:54:29,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742108_1284 (size=12151) 2024-11-07T15:54:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:29,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:29,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994929663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994929666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994929667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994929668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994929669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:54:29,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994929773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994929782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994929783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994929783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994929783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,947 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/051e06a389e9497384994d055e8aa5bf 2024-11-07T15:54:29,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a93174caab7a43d49204a85abe64a094 is 50, key is test_row_0/C:col10/1730994868470/Put/seqid=0 2024-11-07T15:54:29,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742109_1285 (size=12151) 2024-11-07T15:54:29,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994929983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:54:29,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994929993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994929993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994929993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:29,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994929993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994930293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994930303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994930303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994930304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994930304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a93174caab7a43d49204a85abe64a094 2024-11-07T15:54:30,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/78030a7cf39d4b659c24ae4e56836b81 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81 2024-11-07T15:54:30,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T15:54:30,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/051e06a389e9497384994d055e8aa5bf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf 2024-11-07T15:54:30,372 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T15:54:30,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a93174caab7a43d49204a85abe64a094 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094 2024-11-07T15:54:30,376 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094, entries=150, sequenceid=167, filesize=11.9 K 2024-11-07T15:54:30,378 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for c94656b3cdcc591544c8dc0ab185fdfc in 859ms, sequenceid=167, compaction requested=true 2024-11-07T15:54:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:30,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-07T15:54:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-07T15:54:30,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-07T15:54:30,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0130 sec 2024-11-07T15:54:30,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0180 sec 2024-11-07T15:54:30,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-07T15:54:30,504 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:30,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-07T15:54:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:54:30,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:30,506 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:30,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:54:30,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-07T15:54:30,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:30,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:54:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/eda47ebd7e014a749dfb51c76611c805 is 50, key is test_row_0/A:col10/1730994869666/Put/seqid=0 2024-11-07T15:54:30,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742110_1286 (size=12151) 2024-11-07T15:54:30,669 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/eda47ebd7e014a749dfb51c76611c805 2024-11-07T15:54:30,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87e570e308a944c58ac0df2d098e4ec6 is 50, key is test_row_0/B:col10/1730994869666/Put/seqid=0 2024-11-07T15:54:30,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742111_1287 (size=12151) 2024-11-07T15:54:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:30,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:54:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994930831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994930831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994930832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994930833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994930837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994930943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994930943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994930944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994930944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:30,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:30,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994930953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:30,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,080 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87e570e308a944c58ac0df2d098e4ec6 2024-11-07T15:54:31,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c48e8c0b1ecc4be6ad715c0c10e91843 is 50, key is test_row_0/C:col10/1730994869666/Put/seqid=0 2024-11-07T15:54:31,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742112_1288 (size=12151) 2024-11-07T15:54:31,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:54:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994931153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994931153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994931153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994931163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994931163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994931463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994931463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994931463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994931472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994931472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:31,492 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c48e8c0b1ecc4be6ad715c0c10e91843 2024-11-07T15:54:31,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/eda47ebd7e014a749dfb51c76611c805 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805 2024-11-07T15:54:31,501 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805, entries=150, sequenceid=186, filesize=11.9 K 2024-11-07T15:54:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/87e570e308a944c58ac0df2d098e4ec6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6 2024-11-07T15:54:31,505 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6, entries=150, sequenceid=186, filesize=11.9 K 2024-11-07T15:54:31,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/c48e8c0b1ecc4be6ad715c0c10e91843 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843 2024-11-07T15:54:31,510 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843, entries=150, sequenceid=186, filesize=11.9 K 2024-11-07T15:54:31,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for c94656b3cdcc591544c8dc0ab185fdfc in 852ms, sequenceid=186, compaction requested=true 2024-11-07T15:54:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:31,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-07T15:54:31,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-07T15:54:31,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-07T15:54:31,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0050 sec 2024-11-07T15:54:31,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0110 sec 2024-11-07T15:54:31,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-07T15:54:31,643 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:31,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:31,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-07T15:54:31,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:54:31,646 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:31,647 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:31,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:31,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:54:31,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-07T15:54:31,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:31,800 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:54:31,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50c3b024965546cabc4bbac274bb54c4 is 50, key is test_row_0/A:col10/1730994870831/Put/seqid=0 2024-11-07T15:54:31,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742113_1289 (size=12151) 2024-11-07T15:54:31,811 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50c3b024965546cabc4bbac274bb54c4 2024-11-07T15:54:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/1e2b79ba36224a2e8a17c07a53bdc402 is 50, key is test_row_0/B:col10/1730994870831/Put/seqid=0 2024-11-07T15:54:31,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742114_1290 (size=12151) 2024-11-07T15:54:31,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/1e2b79ba36224a2e8a17c07a53bdc402 2024-11-07T15:54:31,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/1d66c3955f4144c88fad6a3cbb4dfe58 is 50, key is test_row_0/C:col10/1730994870831/Put/seqid=0 2024-11-07T15:54:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742115_1291 (size=12151) 2024-11-07T15:54:31,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/1d66c3955f4144c88fad6a3cbb4dfe58 2024-11-07T15:54:31,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/50c3b024965546cabc4bbac274bb54c4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4 2024-11-07T15:54:31,850 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4, entries=150, sequenceid=203, filesize=11.9 K 2024-11-07T15:54:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/1e2b79ba36224a2e8a17c07a53bdc402 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402 2024-11-07T15:54:31,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402, entries=150, sequenceid=203, filesize=11.9 K 2024-11-07T15:54:31,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/1d66c3955f4144c88fad6a3cbb4dfe58 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58 2024-11-07T15:54:31,861 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58, entries=150, sequenceid=203, filesize=11.9 K 2024-11-07T15:54:31,862 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for c94656b3cdcc591544c8dc0ab185fdfc in 62ms, sequenceid=203, compaction requested=true 2024-11-07T15:54:31,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:31,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:31,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-07T15:54:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-07T15:54:31,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-07T15:54:31,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 216 msec 2024-11-07T15:54:31,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 222 msec 2024-11-07T15:54:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-07T15:54:31,964 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:31,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-07T15:54:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:31,966 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:31,967 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:31,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:31,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:31,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:54:31,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/0c2ebe1e6e9944609ee0b496843ada75 is 50, key is test_row_0/A:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:31,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742116_1292 (size=14541) 2024-11-07T15:54:32,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994932060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994932061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:32,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994932069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994932070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994932071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994932173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994932173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994932183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994932183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994932183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:32,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994932383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994932383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,389 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994932392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994932393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994932393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/0c2ebe1e6e9944609ee0b496843ada75 2024-11-07T15:54:32,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/01a8a0403cf64fc1884b14255e8dd06b is 50, key is test_row_0/B:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:32,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742117_1293 (size=12151) 2024-11-07T15:54:32,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994932693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994932693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994932702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994932703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994932703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:32,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/01a8a0403cf64fc1884b14255e8dd06b 2024-11-07T15:54:32,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/9333c53def664fc497b5738bf4afa14b is 50, key is test_row_0/C:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:32,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742118_1294 (size=12151) 2024-11-07T15:54:32,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:32,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:33,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994933203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994933213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994933213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994933213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994933213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:33,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/9333c53def664fc497b5738bf4afa14b 2024-11-07T15:54:33,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/0c2ebe1e6e9944609ee0b496843ada75 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75 2024-11-07T15:54:33,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75, entries=200, sequenceid=214, filesize=14.2 K 2024-11-07T15:54:33,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/01a8a0403cf64fc1884b14255e8dd06b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b 2024-11-07T15:54:33,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-07T15:54:33,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/9333c53def664fc497b5738bf4afa14b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b 2024-11-07T15:54:33,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b, entries=150, sequenceid=214, filesize=11.9 K 2024-11-07T15:54:33,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c94656b3cdcc591544c8dc0ab185fdfc in 1250ms, sequenceid=214, compaction requested=true 2024-11-07T15:54:33,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:33,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:33,240 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 9 store files, 0 compacting, 9 eligible, 16 blocking 2024-11-07T15:54:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:33,240 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 9 store files, 0 compacting, 9 eligible, 16 blocking 2024-11-07T15:54:33,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:33,242 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 9 files of size 111287 starting at candidate #0 after considering 28 permutations with 28 in ratio 2024-11-07T15:54:33,242 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/A is initiating minor compaction (all files) 2024-11-07T15:54:33,242 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/A in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,243 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/fa79e79128ac449e9e00ce61cc243189, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=108.7 K 2024-11-07T15:54:33,243 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 9 files of size 108897 starting at candidate #0 after considering 28 permutations with 28 in ratio 2024-11-07T15:54:33,243 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/B is initiating minor compaction (all files) 2024-11-07T15:54:33,243 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/B in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,243 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/8d9ca673145846b9adbc015414e14b13, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=106.3 K 2024-11-07T15:54:33,243 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa79e79128ac449e9e00ce61cc243189, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:33,243 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d9ca673145846b9adbc015414e14b13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:33,243 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting faa93816ce3546858c1167f9dde019bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994863836 2024-11-07T15:54:33,244 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4763d2179ebe49a9930197a51001b74f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994863836 2024-11-07T15:54:33,244 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b811babe8e73487f9606e08a850d0490, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994865001 2024-11-07T15:54:33,244 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f15e4042b2d4b5c92e5b5b5e868fdf3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994865001 2024-11-07T15:54:33,244 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 458cfea18896468d954cb81180e69b29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730994865135 2024-11-07T15:54:33,244 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b77544c190342bfb81886cf6609b34a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730994865135 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dfc6d54db974ebb8b07df18506abcaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1730994867313 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting df757a1c3ea54c6c97c4927442d0191c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1730994867313 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78030a7cf39d4b659c24ae4e56836b81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730994868470 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 051e06a389e9497384994d055e8aa5bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730994868470 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda47ebd7e014a749dfb51c76611c805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1730994869665 2024-11-07T15:54:33,245 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 87e570e308a944c58ac0df2d098e4ec6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1730994869665 2024-11-07T15:54:33,246 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c3b024965546cabc4bbac274bb54c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730994870830 2024-11-07T15:54:33,246 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2b79ba36224a2e8a17c07a53bdc402, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730994870830 2024-11-07T15:54:33,246 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2ebe1e6e9944609ee0b496843ada75, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:33,246 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a8a0403cf64fc1884b14255e8dd06b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:33,263 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#B#compaction#246 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:33,263 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/0eca74b8a54c4c3889739b6fbd19c45b is 50, key is test_row_0/B:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:33,267 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#A#compaction#247 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:33,267 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/39ee6c6cde9c479b9697efcae6b69c28 is 50, key is test_row_0/A:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:33,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742120_1296 (size=12595) 2024-11-07T15:54:33,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742119_1295 (size=12595) 2024-11-07T15:54:33,287 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/0eca74b8a54c4c3889739b6fbd19c45b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/0eca74b8a54c4c3889739b6fbd19c45b 2024-11-07T15:54:33,292 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 9 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/B of c94656b3cdcc591544c8dc0ab185fdfc into 0eca74b8a54c4c3889739b6fbd19c45b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:33,292 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:33,292 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/B, priority=7, startTime=1730994873240; duration=0sec 2024-11-07T15:54:33,292 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:33,293 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:B 2024-11-07T15:54:33,293 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 9 store files, 0 compacting, 9 eligible, 16 blocking 2024-11-07T15:54:33,295 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 9 files of size 108897 starting at candidate #0 after considering 28 permutations with 28 in ratio 2024-11-07T15:54:33,295 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/C is initiating minor compaction (all files) 2024-11-07T15:54:33,295 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/C in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,296 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/d3b4c99fdea34afd997c02148532fd7c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=106.3 K 2024-11-07T15:54:33,296 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d3b4c99fdea34afd997c02148532fd7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1730994862679 2024-11-07T15:54:33,296 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a842dcc3b1244b9181843247bbe8d3e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1730994863836 2024-11-07T15:54:33,297 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c1a2772f20da461095c85b83fee6ec1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1730994865001 2024-11-07T15:54:33,297 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db23bc474cf4626b8b8793a716fff74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1730994865135 2024-11-07T15:54:33,297 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting cf106d0d0c01409a8e2ce3f56a2d3d67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1730994867313 2024-11-07T15:54:33,298 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a93174caab7a43d49204a85abe64a094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1730994868470 2024-11-07T15:54:33,298 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c48e8c0b1ecc4be6ad715c0c10e91843, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1730994869665 2024-11-07T15:54:33,298 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d66c3955f4144c88fad6a3cbb4dfe58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1730994870830 2024-11-07T15:54:33,299 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9333c53def664fc497b5738bf4afa14b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:33,324 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#C#compaction#248 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:33,325 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a328826d2c0b491c83809cf58fedc179 is 50, key is test_row_0/C:col10/1730994871986/Put/seqid=0 2024-11-07T15:54:33,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-07T15:54:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:33,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:54:33,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742121_1297 (size=12595) 2024-11-07T15:54:33,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/84627e697c18462ebf9ff6ada0eb403d is 50, key is test_row_0/A:col10/1730994872022/Put/seqid=0 2024-11-07T15:54:33,361 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a328826d2c0b491c83809cf58fedc179 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a328826d2c0b491c83809cf58fedc179 2024-11-07T15:54:33,369 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 9 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/C of c94656b3cdcc591544c8dc0ab185fdfc into a328826d2c0b491c83809cf58fedc179(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:33,369 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:33,369 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/C, priority=7, startTime=1730994873240; duration=0sec 2024-11-07T15:54:33,370 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:33,370 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:C 2024-11-07T15:54:33,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742122_1298 (size=12151) 2024-11-07T15:54:33,687 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/39ee6c6cde9c479b9697efcae6b69c28 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/39ee6c6cde9c479b9697efcae6b69c28 2024-11-07T15:54:33,695 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 9 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/A of c94656b3cdcc591544c8dc0ab185fdfc into 39ee6c6cde9c479b9697efcae6b69c28(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:33,695 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:33,695 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/A, priority=7, startTime=1730994873239; duration=0sec 2024-11-07T15:54:33,695 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:33,695 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:A 2024-11-07T15:54:33,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/84627e697c18462ebf9ff6ada0eb403d 2024-11-07T15:54:33,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/504d494a22c84800b0e9fe9183e8fa40 is 50, key is test_row_0/B:col10/1730994872022/Put/seqid=0 2024-11-07T15:54:33,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742123_1299 (size=12151) 2024-11-07T15:54:33,831 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/504d494a22c84800b0e9fe9183e8fa40 2024-11-07T15:54:33,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7c1c0685936c45a097a5e5ef8c8ecfe6 is 50, key is test_row_0/C:col10/1730994872022/Put/seqid=0 2024-11-07T15:54:33,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742124_1300 (size=12151) 2024-11-07T15:54:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:34,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:34,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994934232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994934233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994934233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994934234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,238 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,238 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:34,238 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994934235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7c1c0685936c45a097a5e5ef8c8ecfe6 2024-11-07T15:54:34,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/84627e697c18462ebf9ff6ada0eb403d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d 2024-11-07T15:54:34,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d, entries=150, sequenceid=240, filesize=11.9 K 2024-11-07T15:54:34,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/504d494a22c84800b0e9fe9183e8fa40 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40 2024-11-07T15:54:34,258 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40, entries=150, sequenceid=240, filesize=11.9 K 2024-11-07T15:54:34,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/7c1c0685936c45a097a5e5ef8c8ecfe6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6 2024-11-07T15:54:34,263 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6, entries=150, sequenceid=240, filesize=11.9 K 2024-11-07T15:54:34,264 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c94656b3cdcc591544c8dc0ab185fdfc in 918ms, sequenceid=240, compaction requested=false 2024-11-07T15:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:34,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-07T15:54:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-07T15:54:34,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-07T15:54:34,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2980 sec 2024-11-07T15:54:34,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3040 sec 2024-11-07T15:54:34,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:34,345 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:34,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:54:34,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/a4a5020f6d0747df889ed4327e3da38d is 50, key is test_row_0/A:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742125_1301 (size=14541) 2024-11-07T15:54:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994934381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,386 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994934383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994934386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994934493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994934493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994934494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994934703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994934703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994934703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:34,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:34,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/a4a5020f6d0747df889ed4327e3da38d 2024-11-07T15:54:34,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/f1745e52e47d418c832759d5c854d648 is 50, key is test_row_0/B:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742126_1302 (size=12151) 2024-11-07T15:54:35,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994935013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994935013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994935013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/f1745e52e47d418c832759d5c854d648 2024-11-07T15:54:35,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/6006d5ebb6aa48129122e9f3b75b3703 is 50, key is test_row_0/C:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:35,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742127_1303 (size=12151) 2024-11-07T15:54:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994935523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:35,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:35,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/6006d5ebb6aa48129122e9f3b75b3703 2024-11-07T15:54:35,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/a4a5020f6d0747df889ed4327e3da38d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d 2024-11-07T15:54:35,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d, entries=200, sequenceid=256, filesize=14.2 K 2024-11-07T15:54:35,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/f1745e52e47d418c832759d5c854d648 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648 2024-11-07T15:54:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648, entries=150, sequenceid=256, filesize=11.9 K 2024-11-07T15:54:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/6006d5ebb6aa48129122e9f3b75b3703 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703 2024-11-07T15:54:35,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703, entries=150, sequenceid=256, filesize=11.9 K 2024-11-07T15:54:35,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c94656b3cdcc591544c8dc0ab185fdfc in 1277ms, sequenceid=256, compaction requested=true 2024-11-07T15:54:35,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:35,623 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:35,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:35,624 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:35,624 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/A is initiating minor compaction (all files) 2024-11-07T15:54:35,624 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/A in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:35,624 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/39ee6c6cde9c479b9697efcae6b69c28, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=38.4 K 2024-11-07T15:54:35,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:35,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:35,625 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39ee6c6cde9c479b9697efcae6b69c28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:35,626 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84627e697c18462ebf9ff6ada0eb403d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994872022 2024-11-07T15:54:35,626 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a5020f6d0747df889ed4327e3da38d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:35,627 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:35,627 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/B is initiating minor compaction (all files) 2024-11-07T15:54:35,627 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/B in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:35,627 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/0eca74b8a54c4c3889739b6fbd19c45b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=36.0 K 2024-11-07T15:54:35,628 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eca74b8a54c4c3889739b6fbd19c45b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:35,628 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 504d494a22c84800b0e9fe9183e8fa40, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994872022 2024-11-07T15:54:35,629 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting f1745e52e47d418c832759d5c854d648, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:35,641 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#A#compaction#255 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:35,642 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:35,642 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/a1c2501087f04d81bcffe388122c29b0 is 50, key is test_row_0/B:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:35,642 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/05273576020b48d6b3f120c4a6af16c7 is 50, key is test_row_0/A:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:35,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742128_1304 (size=12697) 2024-11-07T15:54:35,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742129_1305 (size=12697) 2024-11-07T15:54:35,693 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/a1c2501087f04d81bcffe388122c29b0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/a1c2501087f04d81bcffe388122c29b0 2024-11-07T15:54:35,698 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/B of c94656b3cdcc591544c8dc0ab185fdfc into a1c2501087f04d81bcffe388122c29b0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:35,698 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:35,699 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/B, priority=13, startTime=1730994875624; duration=0sec 2024-11-07T15:54:35,699 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:35,699 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:B 2024-11-07T15:54:35,699 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:35,700 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:35,700 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/C is initiating minor compaction (all files) 2024-11-07T15:54:35,700 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/C in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:35,700 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a328826d2c0b491c83809cf58fedc179, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=36.0 K 2024-11-07T15:54:35,701 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a328826d2c0b491c83809cf58fedc179, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1730994871986 2024-11-07T15:54:35,701 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c1c0685936c45a097a5e5ef8c8ecfe6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994872022 2024-11-07T15:54:35,702 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6006d5ebb6aa48129122e9f3b75b3703, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:35,712 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#C#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:35,713 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/2234776ed1fb4539b07cf56b0555ac5e is 50, key is test_row_0/C:col10/1730994874234/Put/seqid=0 2024-11-07T15:54:35,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742130_1306 (size=12697) 2024-11-07T15:54:35,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/2234776ed1fb4539b07cf56b0555ac5e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2234776ed1fb4539b07cf56b0555ac5e 2024-11-07T15:54:35,727 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/C of c94656b3cdcc591544c8dc0ab185fdfc into 2234776ed1fb4539b07cf56b0555ac5e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:35,727 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:35,727 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/C, priority=13, startTime=1730994875625; duration=0sec 2024-11-07T15:54:35,727 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:35,727 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:C 2024-11-07T15:54:36,060 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/05273576020b48d6b3f120c4a6af16c7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/05273576020b48d6b3f120c4a6af16c7 2024-11-07T15:54:36,065 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/A of c94656b3cdcc591544c8dc0ab185fdfc into 05273576020b48d6b3f120c4a6af16c7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:36,065 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:36,065 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/A, priority=13, startTime=1730994875623; duration=0sec 2024-11-07T15:54:36,065 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:36,065 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:A 2024-11-07T15:54:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-07T15:54:36,133 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:36,133 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-07T15:54:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:36,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:36,136 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:36,137 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:36,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:36,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:54:36,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/327a76001ca14ade95ffe6024c6d0268 is 50, key is test_row_0/A:col10/1730994874379/Put/seqid=0 2024-11-07T15:54:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742131_1307 (size=14741) 2024-11-07T15:54:36,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994936279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994936282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:36,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:36,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994936394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994936396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:36,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:36,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994936533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,536 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994936543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994936543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,544 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,545 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:36,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:36,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994936603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994936603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/327a76001ca14ade95ffe6024c6d0268 2024-11-07T15:54:36,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6eb5da493f26424fb9edfb37fadf09e4 is 50, key is test_row_0/B:col10/1730994874379/Put/seqid=0 2024-11-07T15:54:36,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742132_1308 (size=12301) 2024-11-07T15:54:36,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:36,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:36,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:36,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:36,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994936913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:36,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:36,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:37,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:37,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:37,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6eb5da493f26424fb9edfb37fadf09e4 2024-11-07T15:54:37,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/57356004144d447a9aeb88d74fb007d7 is 50, key is test_row_0/C:col10/1730994874379/Put/seqid=0 2024-11-07T15:54:37,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742133_1309 (size=12301) 2024-11-07T15:54:37,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/57356004144d447a9aeb88d74fb007d7 2024-11-07T15:54:37,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/327a76001ca14ade95ffe6024c6d0268 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268 2024-11-07T15:54:37,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268, entries=200, sequenceid=282, filesize=14.4 K 2024-11-07T15:54:37,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/6eb5da493f26424fb9edfb37fadf09e4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4 2024-11-07T15:54:37,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T15:54:37,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/57356004144d447a9aeb88d74fb007d7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7 2024-11-07T15:54:37,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7, entries=150, sequenceid=282, filesize=12.0 K 2024-11-07T15:54:37,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c94656b3cdcc591544c8dc0ab185fdfc in 858ms, sequenceid=282, compaction requested=false 2024-11-07T15:54:37,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:37,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-07T15:54:37,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:37,210 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:54:37,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/4ebdc63fedba460ea8f3caf418249f9f is 50, key is test_row_0/A:col10/1730994876262/Put/seqid=0 2024-11-07T15:54:37,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742134_1310 (size=12301) 2024-11-07T15:54:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:37,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:37,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994937465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994937470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994937573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994937582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,619 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/4ebdc63fedba460ea8f3caf418249f9f 2024-11-07T15:54:37,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/e7eec58ecac04f72a142eb41eb3552c7 is 50, key is test_row_0/B:col10/1730994876262/Put/seqid=0 2024-11-07T15:54:37,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742135_1311 (size=12301) 2024-11-07T15:54:37,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994937783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:37,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994937793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:37,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,031 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/e7eec58ecac04f72a142eb41eb3552c7 2024-11-07T15:54:38,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/43252cf9e4ce4f20a2db485a3da485f4 is 50, key is test_row_0/C:col10/1730994876262/Put/seqid=0 2024-11-07T15:54:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742136_1312 (size=12301) 2024-11-07T15:54:38,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994938092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994938103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:38,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/43252cf9e4ce4f20a2db485a3da485f4 2024-11-07T15:54:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/4ebdc63fedba460ea8f3caf418249f9f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f 2024-11-07T15:54:38,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f, entries=150, sequenceid=295, filesize=12.0 K 2024-11-07T15:54:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/e7eec58ecac04f72a142eb41eb3552c7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7 2024-11-07T15:54:38,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7, entries=150, sequenceid=295, filesize=12.0 K 2024-11-07T15:54:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/43252cf9e4ce4f20a2db485a3da485f4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4 2024-11-07T15:54:38,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4, entries=150, sequenceid=295, filesize=12.0 K 2024-11-07T15:54:38,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c94656b3cdcc591544c8dc0ab185fdfc in 1252ms, sequenceid=295, compaction requested=true 2024-11-07T15:54:38,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:38,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:38,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-07T15:54:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-07T15:54:38,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-07T15:54:38,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-11-07T15:54:38,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.3330 sec 2024-11-07T15:54:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:38,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:38,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:54:38,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/e4e34d19c9034275a88da7b28888ff53 is 50, key is test_row_0/A:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:38,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742137_1313 (size=14741) 2024-11-07T15:54:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994938602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994938603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994938603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994938603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994938613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994938713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994938713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994938713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994938923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994938923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994938923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:38,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:38,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/e4e34d19c9034275a88da7b28888ff53 2024-11-07T15:54:38,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/771c99c5e494487591263d5ec3ce7463 is 50, key is test_row_0/B:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:38,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742138_1314 (size=12301) 2024-11-07T15:54:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994939233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994939233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994939233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/771c99c5e494487591263d5ec3ce7463 2024-11-07T15:54:39,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/95d42c1b8d68499fb6e3722ae0b86b5d is 50, key is test_row_0/C:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:39,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742139_1315 (size=12301) 2024-11-07T15:54:39,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52726 deadline: 1730994939623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52678 deadline: 1730994939623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,626 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,627 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in c94656b3cdcc591544c8dc0ab185fdfc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52676 deadline: 1730994939743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52692 deadline: 1730994939743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52740 deadline: 1730994939743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:54:39,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/95d42c1b8d68499fb6e3722ae0b86b5d 2024-11-07T15:54:39,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/e4e34d19c9034275a88da7b28888ff53 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53 2024-11-07T15:54:39,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53, entries=200, sequenceid=319, filesize=14.4 K 2024-11-07T15:54:39,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/771c99c5e494487591263d5ec3ce7463 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463 2024-11-07T15:54:39,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T15:54:39,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/95d42c1b8d68499fb6e3722ae0b86b5d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d 2024-11-07T15:54:39,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d, entries=150, sequenceid=319, filesize=12.0 K 2024-11-07T15:54:39,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c94656b3cdcc591544c8dc0ab185fdfc in 1262ms, sequenceid=319, compaction requested=true 2024-11-07T15:54:39,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:39,820 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c94656b3cdcc591544c8dc0ab185fdfc:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:39,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:39,820 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:39,821 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:39,821 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:39,821 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/B is initiating minor compaction (all files) 2024-11-07T15:54:39,821 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/B in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:39,821 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/a1c2501087f04d81bcffe388122c29b0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=48.4 K 2024-11-07T15:54:39,821 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/A is initiating minor compaction (all files) 2024-11-07T15:54:39,822 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/A in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:39,822 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/05273576020b48d6b3f120c4a6af16c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=53.2 K 2024-11-07T15:54:39,822 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c2501087f04d81bcffe388122c29b0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:39,822 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05273576020b48d6b3f120c4a6af16c7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327a76001ca14ade95ffe6024c6d0268, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730994874379 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb5da493f26424fb9edfb37fadf09e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730994874379 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ebdc63fedba460ea8f3caf418249f9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994876262 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e7eec58ecac04f72a142eb41eb3552c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994876262 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4e34d19c9034275a88da7b28888ff53, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730994877456 2024-11-07T15:54:39,823 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 771c99c5e494487591263d5ec3ce7463, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730994877456 2024-11-07T15:54:39,833 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#A#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:39,833 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#B#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:39,834 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/7b5f9856b1264b0d9937fd39604b3875 is 50, key is test_row_0/B:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:39,834 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/f8f70c934c3f480cb928cbe5c6d1bee3 is 50, key is test_row_0/A:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:39,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742140_1316 (size=12983) 2024-11-07T15:54:39,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742141_1317 (size=12983) 2024-11-07T15:54:40,249 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/f8f70c934c3f480cb928cbe5c6d1bee3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f8f70c934c3f480cb928cbe5c6d1bee3 2024-11-07T15:54:40,250 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/7b5f9856b1264b0d9937fd39604b3875 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/7b5f9856b1264b0d9937fd39604b3875 2024-11-07T15:54:40,261 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/B of c94656b3cdcc591544c8dc0ab185fdfc into 7b5f9856b1264b0d9937fd39604b3875(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:40,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:40,261 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/B, priority=12, startTime=1730994879820; duration=0sec 2024-11-07T15:54:40,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:40,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:B 2024-11-07T15:54:40,261 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:54:40,262 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:54:40,262 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): c94656b3cdcc591544c8dc0ab185fdfc/C is initiating minor compaction (all files) 2024-11-07T15:54:40,262 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c94656b3cdcc591544c8dc0ab185fdfc/C in TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:40,262 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2234776ed1fb4539b07cf56b0555ac5e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp, totalSize=48.4 K 2024-11-07T15:54:40,263 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2234776ed1fb4539b07cf56b0555ac5e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1730994874231 2024-11-07T15:54:40,263 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 57356004144d447a9aeb88d74fb007d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1730994874379 2024-11-07T15:54:40,264 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 43252cf9e4ce4f20a2db485a3da485f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1730994876262 2024-11-07T15:54:40,264 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/A of c94656b3cdcc591544c8dc0ab185fdfc into f8f70c934c3f480cb928cbe5c6d1bee3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:40,264 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:40,264 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/A, priority=12, startTime=1730994879820; duration=0sec 2024-11-07T15:54:40,264 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:40,264 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:A 2024-11-07T15:54:40,265 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d42c1b8d68499fb6e3722ae0b86b5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1730994877456 2024-11-07T15:54:40,274 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c94656b3cdcc591544c8dc0ab185fdfc#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:40,274 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/0bccf34dc951460484261b45b95466b0 is 50, key is test_row_0/C:col10/1730994878555/Put/seqid=0 2024-11-07T15:54:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742142_1318 (size=12983) 2024-11-07T15:54:40,286 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/0bccf34dc951460484261b45b95466b0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/0bccf34dc951460484261b45b95466b0 2024-11-07T15:54:40,294 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c94656b3cdcc591544c8dc0ab185fdfc/C of c94656b3cdcc591544c8dc0ab185fdfc into 0bccf34dc951460484261b45b95466b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:40,294 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:40,294 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., storeName=c94656b3cdcc591544c8dc0ab185fdfc/C, priority=12, startTime=1730994879820; duration=0sec 2024-11-07T15:54:40,294 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:40,294 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c94656b3cdcc591544c8dc0ab185fdfc:C 2024-11-07T15:54:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-07T15:54:40,304 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:40,304 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-07T15:54:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:54:40,307 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:40,308 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:40,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:54:40,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-07T15:54:40,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:40,462 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:54:40,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/f4008f6265c7438d97f6d7f587b1c74f is 50, key is test_row_0/A:col10/1730994878570/Put/seqid=0 2024-11-07T15:54:40,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742143_1319 (size=12301) 2024-11-07T15:54:40,514 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/f4008f6265c7438d97f6d7f587b1c74f 2024-11-07T15:54:40,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/2f71c2b5d45146599dae8b30e8a1870b is 50, key is test_row_0/B:col10/1730994878570/Put/seqid=0 2024-11-07T15:54:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742144_1320 (size=12301) 2024-11-07T15:54:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:54:40,753 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-11-07T15:54:40,753 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,753 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,753 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,754 INFO [Thread-1197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1197. 2024-11-07T15:54:40,754 DEBUG [Thread-1197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,754 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,755 INFO [Thread-1199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1199. 2024-11-07T15:54:40,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,755 DEBUG [Thread-1199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,755 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,755 DEBUG [Thread-1197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,756 INFO [Thread-1195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1195. 2024-11-07T15:54:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:40,756 DEBUG [Thread-1195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. as already flushing 2024-11-07T15:54:40,756 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,756 DEBUG [Thread-1199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,756 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-07T15:54:40,756 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,756 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,756 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,756 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-11-07T15:54:40,757 DEBUG [Thread-1195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,757 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,757 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,757 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,757 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-07T15:54:40,757 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,758 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,758 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,758 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-07T15:54:40,758 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:40,758 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,758 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:40,758 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:54:40,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/2f71c2b5d45146599dae8b30e8a1870b 2024-11-07T15:54:40,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/4f187c7b7c844886b5c716dea6f5b716 is 50, key is test_row_0/C:col10/1730994878570/Put/seqid=0 2024-11-07T15:54:40,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742145_1321 (size=12301) 2024-11-07T15:54:41,346 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/4f187c7b7c844886b5c716dea6f5b716 2024-11-07T15:54:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/f4008f6265c7438d97f6d7f587b1c74f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f4008f6265c7438d97f6d7f587b1c74f 2024-11-07T15:54:41,353 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f4008f6265c7438d97f6d7f587b1c74f, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T15:54:41,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/2f71c2b5d45146599dae8b30e8a1870b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/2f71c2b5d45146599dae8b30e8a1870b 2024-11-07T15:54:41,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/2f71c2b5d45146599dae8b30e8a1870b, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T15:54:41,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/4f187c7b7c844886b5c716dea6f5b716 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/4f187c7b7c844886b5c716dea6f5b716 2024-11-07T15:54:41,360 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/4f187c7b7c844886b5c716dea6f5b716, entries=150, sequenceid=334, filesize=12.0 K 2024-11-07T15:54:41,361 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=20.13 KB/20610 for c94656b3cdcc591544c8dc0ab185fdfc in 900ms, sequenceid=334, compaction requested=false 2024-11-07T15:54:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for c94656b3cdcc591544c8dc0ab185fdfc: 2024-11-07T15:54:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:41,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-07T15:54:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-07T15:54:41,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-07T15:54:41,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0540 sec 2024-11-07T15:54:41,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.0600 sec 2024-11-07T15:54:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-07T15:54:41,443 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:41,635 INFO [Thread-1180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1180. 2024-11-07T15:54:41,635 DEBUG [Thread-1180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:41,635 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,635 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,635 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:41,644 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-07T15:54:41,644 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:54:41,644 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,644 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,644 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3237 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3263 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3243 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3253 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3267 2024-11-07T15:54:41,644 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:54:41,644 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:54:41,645 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:54:41,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:41,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:54:41,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:54:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:41,649 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994881648"}]},"ts":"1730994881648"} 2024-11-07T15:54:41,650 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:54:41,650 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:54:41,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:54:41,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, UNASSIGN}] 2024-11-07T15:54:41,653 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, UNASSIGN 2024-11-07T15:54:41,653 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c94656b3cdcc591544c8dc0ab185fdfc, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:41,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, UNASSIGN because future has completed 2024-11-07T15:54:41,654 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:54:41,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:41,807 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:41,807 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:54:41,807 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing c94656b3cdcc591544c8dc0ab185fdfc, disabling compactions & flushes 2024-11-07T15:54:41,807 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:41,807 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:41,807 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. after waiting 0 ms 2024-11-07T15:54:41,807 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:41,807 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing c94656b3cdcc591544c8dc0ab185fdfc 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:54:41,811 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/277e173b369e4dfdad4d113fca32e9f9 is 50, key is test_row_0/A:col10/1730994881643/Put/seqid=0 2024-11-07T15:54:41,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742146_1322 (size=12301) 2024-11-07T15:54:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:42,215 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/277e173b369e4dfdad4d113fca32e9f9 2024-11-07T15:54:42,222 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/13eb415c0b8b4de58075607e4d31d352 is 50, key is test_row_0/B:col10/1730994881643/Put/seqid=0 2024-11-07T15:54:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742147_1323 (size=12301) 2024-11-07T15:54:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:42,625 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/13eb415c0b8b4de58075607e4d31d352 2024-11-07T15:54:42,632 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a162b62a9ed348e0976555f704b05cc9 is 50, key is test_row_0/C:col10/1730994881643/Put/seqid=0 2024-11-07T15:54:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742148_1324 (size=12301) 2024-11-07T15:54:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:43,036 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a162b62a9ed348e0976555f704b05cc9 2024-11-07T15:54:43,040 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/A/277e173b369e4dfdad4d113fca32e9f9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/277e173b369e4dfdad4d113fca32e9f9 2024-11-07T15:54:43,043 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/277e173b369e4dfdad4d113fca32e9f9, entries=150, sequenceid=342, filesize=12.0 K 2024-11-07T15:54:43,043 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/B/13eb415c0b8b4de58075607e4d31d352 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/13eb415c0b8b4de58075607e4d31d352 2024-11-07T15:54:43,046 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/13eb415c0b8b4de58075607e4d31d352, entries=150, sequenceid=342, filesize=12.0 K 2024-11-07T15:54:43,047 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/.tmp/C/a162b62a9ed348e0976555f704b05cc9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a162b62a9ed348e0976555f704b05cc9 2024-11-07T15:54:43,049 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a162b62a9ed348e0976555f704b05cc9, entries=150, sequenceid=342, filesize=12.0 K 2024-11-07T15:54:43,050 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c94656b3cdcc591544c8dc0ab185fdfc in 1243ms, sequenceid=342, compaction requested=true 2024-11-07T15:54:43,051 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/fa79e79128ac449e9e00ce61cc243189, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/39ee6c6cde9c479b9697efcae6b69c28, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/05273576020b48d6b3f120c4a6af16c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53] to archive 2024-11-07T15:54:43,051 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:43,052 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50814cfbd9514adbb39e4fa483f793f2 2024-11-07T15:54:43,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/14000e0bbead456ea8dad6a400b845e6 2024-11-07T15:54:43,054 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/38a0123278564425af0d6457078f8655 2024-11-07T15:54:43,055 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/8184c0638c0a4b4fa38e2fd18fa655f9 2024-11-07T15:54:43,056 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/fa79e79128ac449e9e00ce61cc243189 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/fa79e79128ac449e9e00ce61cc243189 2024-11-07T15:54:43,057 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/faa93816ce3546858c1167f9dde019bb 2024-11-07T15:54:43,058 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/b811babe8e73487f9606e08a850d0490 2024-11-07T15:54:43,059 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/458cfea18896468d954cb81180e69b29 2024-11-07T15:54:43,060 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/6dfc6d54db974ebb8b07df18506abcaa 2024-11-07T15:54:43,060 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/78030a7cf39d4b659c24ae4e56836b81 2024-11-07T15:54:43,061 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/eda47ebd7e014a749dfb51c76611c805 2024-11-07T15:54:43,062 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/50c3b024965546cabc4bbac274bb54c4 2024-11-07T15:54:43,063 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/0c2ebe1e6e9944609ee0b496843ada75 2024-11-07T15:54:43,064 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/39ee6c6cde9c479b9697efcae6b69c28 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/39ee6c6cde9c479b9697efcae6b69c28 2024-11-07T15:54:43,065 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/84627e697c18462ebf9ff6ada0eb403d 2024-11-07T15:54:43,066 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/a4a5020f6d0747df889ed4327e3da38d 2024-11-07T15:54:43,067 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/05273576020b48d6b3f120c4a6af16c7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/05273576020b48d6b3f120c4a6af16c7 2024-11-07T15:54:43,068 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/327a76001ca14ade95ffe6024c6d0268 2024-11-07T15:54:43,068 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/4ebdc63fedba460ea8f3caf418249f9f 2024-11-07T15:54:43,069 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/e4e34d19c9034275a88da7b28888ff53 2024-11-07T15:54:43,071 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/8d9ca673145846b9adbc015414e14b13, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/0eca74b8a54c4c3889739b6fbd19c45b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/a1c2501087f04d81bcffe388122c29b0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463] to archive 2024-11-07T15:54:43,071 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:43,072 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87a5c3067fd043bb95e0e23d6702060d 2024-11-07T15:54:43,073 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/cb0909c44db646239a7d203f25ed2f40 2024-11-07T15:54:43,074 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/d9aa871935c54a238ec0b770823e7eb4 2024-11-07T15:54:43,075 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/8d9ca673145846b9adbc015414e14b13 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/8d9ca673145846b9adbc015414e14b13 2024-11-07T15:54:43,076 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/61e7ab4bd3ca435d8740165eeadb44fa 2024-11-07T15:54:43,077 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/4763d2179ebe49a9930197a51001b74f 2024-11-07T15:54:43,078 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6f15e4042b2d4b5c92e5b5b5e868fdf3 2024-11-07T15:54:43,079 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/3b77544c190342bfb81886cf6609b34a 2024-11-07T15:54:43,079 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/df757a1c3ea54c6c97c4927442d0191c 2024-11-07T15:54:43,080 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/051e06a389e9497384994d055e8aa5bf 2024-11-07T15:54:43,081 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/87e570e308a944c58ac0df2d098e4ec6 2024-11-07T15:54:43,082 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/1e2b79ba36224a2e8a17c07a53bdc402 2024-11-07T15:54:43,083 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/0eca74b8a54c4c3889739b6fbd19c45b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/0eca74b8a54c4c3889739b6fbd19c45b 2024-11-07T15:54:43,083 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/01a8a0403cf64fc1884b14255e8dd06b 2024-11-07T15:54:43,084 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/504d494a22c84800b0e9fe9183e8fa40 2024-11-07T15:54:43,085 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/a1c2501087f04d81bcffe388122c29b0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/a1c2501087f04d81bcffe388122c29b0 2024-11-07T15:54:43,086 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/f1745e52e47d418c832759d5c854d648 2024-11-07T15:54:43,086 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/6eb5da493f26424fb9edfb37fadf09e4 2024-11-07T15:54:43,087 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/e7eec58ecac04f72a142eb41eb3552c7 2024-11-07T15:54:43,088 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/771c99c5e494487591263d5ec3ce7463 2024-11-07T15:54:43,089 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/d3b4c99fdea34afd997c02148532fd7c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a328826d2c0b491c83809cf58fedc179, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2234776ed1fb4539b07cf56b0555ac5e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d] to archive 2024-11-07T15:54:43,090 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:54:43,091 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/3566854f37464e7d96b4c17849c2873c 2024-11-07T15:54:43,091 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/db574cd3c94f4418b1c34adcb5f4b629 2024-11-07T15:54:43,092 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7cb8ba761b0c4923a309c82b08966806 2024-11-07T15:54:43,093 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/d3b4c99fdea34afd997c02148532fd7c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/d3b4c99fdea34afd997c02148532fd7c 2024-11-07T15:54:43,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/f7ae640585cc4a5abeb6d1887b98eee6 2024-11-07T15:54:43,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a842dcc3b1244b9181843247bbe8d3e8 2024-11-07T15:54:43,095 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c1a2772f20da461095c85b83fee6ec1c 2024-11-07T15:54:43,096 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2db23bc474cf4626b8b8793a716fff74 2024-11-07T15:54:43,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/cf106d0d0c01409a8e2ce3f56a2d3d67 2024-11-07T15:54:43,098 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a93174caab7a43d49204a85abe64a094 2024-11-07T15:54:43,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/c48e8c0b1ecc4be6ad715c0c10e91843 2024-11-07T15:54:43,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/1d66c3955f4144c88fad6a3cbb4dfe58 2024-11-07T15:54:43,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a328826d2c0b491c83809cf58fedc179 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a328826d2c0b491c83809cf58fedc179 2024-11-07T15:54:43,101 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/9333c53def664fc497b5738bf4afa14b 2024-11-07T15:54:43,102 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/7c1c0685936c45a097a5e5ef8c8ecfe6 2024-11-07T15:54:43,102 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2234776ed1fb4539b07cf56b0555ac5e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/2234776ed1fb4539b07cf56b0555ac5e 2024-11-07T15:54:43,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/6006d5ebb6aa48129122e9f3b75b3703 2024-11-07T15:54:43,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/57356004144d447a9aeb88d74fb007d7 2024-11-07T15:54:43,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/43252cf9e4ce4f20a2db485a3da485f4 2024-11-07T15:54:43,106 DEBUG [StoreCloser-TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/95d42c1b8d68499fb6e3722ae0b86b5d 2024-11-07T15:54:43,109 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=1 2024-11-07T15:54:43,110 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. 2024-11-07T15:54:43,110 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for c94656b3cdcc591544c8dc0ab185fdfc: Waiting for close lock at 1730994881807Running coprocessor pre-close hooks at 1730994881807Disabling compacts and flushes for region at 1730994881807Disabling writes for close at 1730994881807Obtaining lock to block concurrent updates at 1730994881807Preparing flush snapshotting stores in c94656b3cdcc591544c8dc0ab185fdfc at 1730994881807Finished memstore snapshotting TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730994881808 (+1 ms)Flushing stores of TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc. at 1730994881808Flushing c94656b3cdcc591544c8dc0ab185fdfc/A: creating writer at 1730994881808Flushing c94656b3cdcc591544c8dc0ab185fdfc/A: appending metadata at 1730994881811 (+3 ms)Flushing c94656b3cdcc591544c8dc0ab185fdfc/A: closing flushed file at 1730994881811Flushing c94656b3cdcc591544c8dc0ab185fdfc/B: creating writer at 1730994882219 (+408 ms)Flushing c94656b3cdcc591544c8dc0ab185fdfc/B: appending metadata at 1730994882221 (+2 ms)Flushing c94656b3cdcc591544c8dc0ab185fdfc/B: closing flushed file at 1730994882221Flushing c94656b3cdcc591544c8dc0ab185fdfc/C: creating writer at 1730994882629 (+408 ms)Flushing c94656b3cdcc591544c8dc0ab185fdfc/C: appending metadata at 1730994882631 (+2 ms)Flushing c94656b3cdcc591544c8dc0ab185fdfc/C: closing flushed file at 1730994882631Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1392068d: reopening flushed file at 1730994883039 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19539198: reopening flushed file at 1730994883043 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b5b6e5b: reopening flushed file at 1730994883046 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for c94656b3cdcc591544c8dc0ab185fdfc in 1243ms, sequenceid=342, compaction requested=true at 1730994883050 (+4 ms)Writing region close event to WAL at 1730994883107 (+57 ms)Running coprocessor post-close hooks at 1730994883110 (+3 ms)Closed at 1730994883110 2024-11-07T15:54:43,112 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:43,112 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c94656b3cdcc591544c8dc0ab185fdfc, regionState=CLOSED 2024-11-07T15:54:43,113 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:43,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-07T15:54:43,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure c94656b3cdcc591544c8dc0ab185fdfc, server=c2682f91c1e2,42309,1730994796599 in 1.4590 sec 2024-11-07T15:54:43,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-07T15:54:43,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c94656b3cdcc591544c8dc0ab185fdfc, UNASSIGN in 1.4630 sec 2024-11-07T15:54:43,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-07T15:54:43,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-11-07T15:54:43,120 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994883120"}]},"ts":"1730994883120"} 2024-11-07T15:54:43,121 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:54:43,121 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:54:43,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4770 sec 2024-11-07T15:54:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-07T15:54:43,793 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:43,793 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:54:43,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,795 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T15:54:43,796 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,798 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:43,800 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/recovered.edits] 2024-11-07T15:54:43,802 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/277e173b369e4dfdad4d113fca32e9f9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/277e173b369e4dfdad4d113fca32e9f9 2024-11-07T15:54:43,803 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f4008f6265c7438d97f6d7f587b1c74f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f4008f6265c7438d97f6d7f587b1c74f 2024-11-07T15:54:43,804 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f8f70c934c3f480cb928cbe5c6d1bee3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/A/f8f70c934c3f480cb928cbe5c6d1bee3 2024-11-07T15:54:43,806 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/13eb415c0b8b4de58075607e4d31d352 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/13eb415c0b8b4de58075607e4d31d352 2024-11-07T15:54:43,807 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/2f71c2b5d45146599dae8b30e8a1870b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/2f71c2b5d45146599dae8b30e8a1870b 2024-11-07T15:54:43,808 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/7b5f9856b1264b0d9937fd39604b3875 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/B/7b5f9856b1264b0d9937fd39604b3875 2024-11-07T15:54:43,810 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/0bccf34dc951460484261b45b95466b0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/0bccf34dc951460484261b45b95466b0 2024-11-07T15:54:43,810 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/4f187c7b7c844886b5c716dea6f5b716 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/4f187c7b7c844886b5c716dea6f5b716 2024-11-07T15:54:43,811 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a162b62a9ed348e0976555f704b05cc9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/C/a162b62a9ed348e0976555f704b05cc9 2024-11-07T15:54:43,814 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/recovered.edits/345.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc/recovered.edits/345.seqid 2024-11-07T15:54:43,814 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/c94656b3cdcc591544c8dc0ab185fdfc 2024-11-07T15:54:43,814 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:54:43,816 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,819 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:54:43,820 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:54:43,821 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,821 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:54:43,821 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994883821"}]},"ts":"9223372036854775807"} 2024-11-07T15:54:43,823 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:54:43,823 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c94656b3cdcc591544c8dc0ab185fdfc, NAME => 'TestAcidGuarantees,,1730994858557.c94656b3cdcc591544c8dc0ab185fdfc.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:54:43,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:54:43,823 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994883823"}]},"ts":"9223372036854775807"} 2024-11-07T15:54:43,825 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:54:43,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-07T15:54:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-07T15:54:43,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:54:43,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:43,915 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 241), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 260) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5570 (was 5656) 2024-11-07T15:54:43,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=5570 2024-11-07T15:54:43,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:54:43,927 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:54:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:43,928 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:54:43,929 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:43,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-07T15:54:43,929 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:54:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:43,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742149_1325 (size=959) 2024-11-07T15:54:44,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:44,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:54:44,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742150_1326 (size=53) 2024-11-07T15:54:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 43137c94133f8cabe05776420e1d74bb, disabling compactions & flushes 2024-11-07T15:54:44,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. after waiting 0 ms 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:44,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:44,742 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 43137c94133f8cabe05776420e1d74bb: Waiting for close lock at 1730994884742Disabling compacts and flushes for region at 1730994884742Disabling writes for close at 1730994884742Writing region close event to WAL at 1730994884742Closed at 1730994884742 2024-11-07T15:54:44,744 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:54:44,744 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994884744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994884744"}]},"ts":"1730994884744"} 2024-11-07T15:54:44,746 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:54:44,747 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:54:44,747 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994884747"}]},"ts":"1730994884747"} 2024-11-07T15:54:44,748 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:54:44,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, ASSIGN}] 2024-11-07T15:54:44,750 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, ASSIGN 2024-11-07T15:54:44,750 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:54:44,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:44,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, ASSIGN because future has completed 2024-11-07T15:54:44,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:44,925 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:54:45,057 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:45,057 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:54:45,058 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,058 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:54:45,058 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,058 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,059 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,060 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName A 2024-11-07T15:54:45,060 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:45,061 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:45,061 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,062 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName B 2024-11-07T15:54:45,062 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:45,062 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:45,062 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:45,063 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName C 2024-11-07T15:54:45,063 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:45,063 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:45,064 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,064 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,064 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,065 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,065 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,066 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:54:45,066 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,068 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:54:45,068 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 43137c94133f8cabe05776420e1d74bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64353553, jitterRate=-0.04105733335018158}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:54:45,068 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:45,069 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 43137c94133f8cabe05776420e1d74bb: Running coprocessor pre-open hook at 1730994885058Writing region info on filesystem at 1730994885058Initializing all the Stores at 1730994885059 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994885059Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994885059Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994885059Cleaning up temporary data from old regions at 1730994885065 (+6 ms)Running coprocessor post-open hooks at 1730994885068 (+3 ms)Region opened successfully at 1730994885069 (+1 ms) 2024-11-07T15:54:45,069 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., pid=93, masterSystemTime=1730994885054 2024-11-07T15:54:45,071 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:45,071 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:45,071 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:45,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:45,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-07T15:54:45,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 in 170 msec 2024-11-07T15:54:45,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-07T15:54:45,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, ASSIGN in 327 msec 2024-11-07T15:54:45,077 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:54:45,077 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994885077"}]},"ts":"1730994885077"} 2024-11-07T15:54:45,078 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:54:45,079 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:54:45,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-07T15:54:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-07T15:54:46,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:46,075 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,075 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:46,075 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:46,076 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:46,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259aef94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:46,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:46,078 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754ecee5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,079 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:46,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:46,081 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:54:46,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:54:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bfaafa7 2024-11-07T15:54:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:54:46,082 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:54:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:54:46,084 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:54:46,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:54:46,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742151_1327 (size=995) 2024-11-07T15:54:46,495 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:54:46,495 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:54:46,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:54:46,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE}] 2024-11-07T15:54:46,500 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE 2024-11-07T15:54:46,500 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:46,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE because future has completed 2024-11-07T15:54:46,502 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:54:46,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:46,654 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,654 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:54:46,654 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 43137c94133f8cabe05776420e1d74bb, disabling compactions & flushes 2024-11-07T15:54:46,654 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,654 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,654 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. after waiting 0 ms 2024-11-07T15:54:46,654 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,658 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:54:46,658 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,658 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 43137c94133f8cabe05776420e1d74bb: Waiting for close lock at 1730994886654Running coprocessor pre-close hooks at 1730994886654Disabling compacts and flushes for region at 1730994886654Disabling writes for close at 1730994886654Writing region close event to WAL at 1730994886655 (+1 ms)Running coprocessor post-close hooks at 1730994886658 (+3 ms)Closed at 1730994886658 2024-11-07T15:54:46,658 WARN [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 43137c94133f8cabe05776420e1d74bb to self. 2024-11-07T15:54:46,660 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,660 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=CLOSED 2024-11-07T15:54:46,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:46,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-07T15:54:46,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 in 160 msec 2024-11-07T15:54:46,664 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE; state=CLOSED, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=true 2024-11-07T15:54:46,814 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:46,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE because future has completed 2024-11-07T15:54:46,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:54:46,971 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,971 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:54:46,971 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,971 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:54:46,971 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,971 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,972 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,973 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName A 2024-11-07T15:54:46,974 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:46,974 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:46,975 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,975 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName B 2024-11-07T15:54:46,975 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:46,976 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:46,976 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,976 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 43137c94133f8cabe05776420e1d74bb columnFamilyName C 2024-11-07T15:54:46,976 DEBUG [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:46,976 INFO [StoreOpener-43137c94133f8cabe05776420e1d74bb-1 {}] regionserver.HStore(327): Store=43137c94133f8cabe05776420e1d74bb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:54:46,977 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,977 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,978 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,979 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,979 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,979 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:54:46,981 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,981 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 43137c94133f8cabe05776420e1d74bb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62911181, jitterRate=-0.06255035102367401}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:54:46,981 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:46,982 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 43137c94133f8cabe05776420e1d74bb: Running coprocessor pre-open hook at 1730994886971Writing region info on filesystem at 1730994886971Initializing all the Stores at 1730994886972 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994886972Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994886972Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994886972Cleaning up temporary data from old regions at 1730994886979 (+7 ms)Running coprocessor post-open hooks at 1730994886981 (+2 ms)Region opened successfully at 1730994886982 (+1 ms) 2024-11-07T15:54:46,983 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., pid=98, masterSystemTime=1730994886968 2024-11-07T15:54:46,985 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,985 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:46,985 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=OPEN, openSeqNum=5, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:46,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:54:46,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-07T15:54:46,989 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 in 171 msec 2024-11-07T15:54:46,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-07T15:54:46,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, REOPEN/MOVE in 490 msec 2024-11-07T15:54:46,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-07T15:54:46,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-07T15:54:46,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-07T15:54:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-07T15:54:46,995 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:46,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:46,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:46,997 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd4b4ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:46,997 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:46,998 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:46,998 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fbc1bca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:46,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:46,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,000 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767f48bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,001 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c135bc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259840a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,004 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61fec268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,005 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d3372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,006 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58706, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26278eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,007 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,007 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,008 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bcbc0eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,009 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28429aa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,010 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,011 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d914fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c5ee299, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,013 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56e07521, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,014 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58758, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed8836b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254240f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,017 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bc3f6ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,019 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f99eab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,019 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,020 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4162cd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:54:47,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:54:47,022 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc21a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:54:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:54:47,023 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:54:47,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@608204b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:54:47,024 DEBUG [Thread-1507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,024 DEBUG [Thread-1513 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,024 DEBUG [Thread-1498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,024 DEBUG [Thread-1515 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,024 DEBUG [Thread-1511 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,025 DEBUG [Thread-1509 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,026 DEBUG [Thread-1502 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,026 DEBUG [Thread-1500 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,026 DEBUG [Thread-1496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,029 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,029 DEBUG [Thread-1504 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:54:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:54:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:54:47,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:54:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,035 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:54:47,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:47,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:47,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:54:47,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:47,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-07T15:54:47,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:47,052 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:47,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:47,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072d867173520d484d8e6cd0abe57fb074_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994887045/Put/seqid=0 2024-11-07T15:54:47,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742152_1328 (size=17034) 2024-11-07T15:54:47,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994947089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994947099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994947099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994947100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994947101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:47,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994947203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994947212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994947223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994947223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994947223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:47,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994947423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994947423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994947443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994947443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994947443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,494 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:47,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072d867173520d484d8e6cd0abe57fb074_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072d867173520d484d8e6cd0abe57fb074_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:47,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/413fc05eb97c4517b9ff4f2c9a3aff2e, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:47,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/413fc05eb97c4517b9ff4f2c9a3aff2e is 175, key is test_row_0/A:col10/1730994887045/Put/seqid=0 2024-11-07T15:54:47,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742153_1329 (size=48139) 2024-11-07T15:54:47,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:47,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994947733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994947733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994947753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994947753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:47,760 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994947753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:47,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:47,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/413fc05eb97c4517b9ff4f2c9a3aff2e 2024-11-07T15:54:47,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/9ae1937152e949bba1bf8bc2e7afbb0e is 50, key is test_row_0/B:col10/1730994887045/Put/seqid=0 2024-11-07T15:54:47,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742154_1330 (size=12001) 2024-11-07T15:54:47,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:47,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:47,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994948253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994948253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994948273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:48,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994948273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:48,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:48,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994948273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:48,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:48,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:48,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/9ae1937152e949bba1bf8bc2e7afbb0e 2024-11-07T15:54:48,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/042fdac84e0a4d1e9c45313b0a731e23 is 50, key is test_row_0/C:col10/1730994887045/Put/seqid=0 2024-11-07T15:54:48,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742155_1331 (size=12001) 2024-11-07T15:54:48,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,634 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:54:48,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:48,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/042fdac84e0a4d1e9c45313b0a731e23 2024-11-07T15:54:48,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/413fc05eb97c4517b9ff4f2c9a3aff2e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e 2024-11-07T15:54:48,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e, entries=250, sequenceid=17, filesize=47.0 K 2024-11-07T15:54:48,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/9ae1937152e949bba1bf8bc2e7afbb0e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e 2024-11-07T15:54:48,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T15:54:48,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/042fdac84e0a4d1e9c45313b0a731e23 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23 2024-11-07T15:54:48,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23, entries=150, sequenceid=17, filesize=11.7 K 2024-11-07T15:54:48,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 43137c94133f8cabe05776420e1d74bb in 1741ms, sequenceid=17, compaction requested=false 2024-11-07T15:54:48,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:48,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-07T15:54:48,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:48,892 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:54:48,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d7db2238c80a470b892317b7d5e0e5e2_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994887098/Put/seqid=0 2024-11-07T15:54:48,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742156_1332 (size=12154) 2024-11-07T15:54:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:49,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994949293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994949293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994949293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,302 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:49,302 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:49,302 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:49,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994949296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994949301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,310 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d7db2238c80a470b892317b7d5e0e5e2_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7db2238c80a470b892317b7d5e0e5e2_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:49,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/cf485b90f08a42a5aa35e0d9797b4c14, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:49,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/cf485b90f08a42a5aa35e0d9797b4c14 is 175, key is test_row_0/A:col10/1730994887098/Put/seqid=0 2024-11-07T15:54:49,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742157_1333 (size=30955) 2024-11-07T15:54:49,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994949413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994949413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994949623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994949623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,716 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/cf485b90f08a42a5aa35e0d9797b4c14 2024-11-07T15:54:49,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/25ec50bc15dc4cf589c9f90712252574 is 50, key is test_row_0/B:col10/1730994887098/Put/seqid=0 2024-11-07T15:54:49,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742158_1334 (size=12001) 2024-11-07T15:54:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994949933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994949933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:49,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:49,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:50,128 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/25ec50bc15dc4cf589c9f90712252574 2024-11-07T15:54:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/fed1ee84284548a1baef5dfb5f720f1f is 50, key is test_row_0/C:col10/1730994887098/Put/seqid=0 2024-11-07T15:54:50,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742159_1335 (size=12001) 2024-11-07T15:54:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994950453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:50,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994950453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:50,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:50,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:50,540 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/fed1ee84284548a1baef5dfb5f720f1f 2024-11-07T15:54:50,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/cf485b90f08a42a5aa35e0d9797b4c14 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14 2024-11-07T15:54:50,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14, entries=150, sequenceid=41, filesize=30.2 K 2024-11-07T15:54:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/25ec50bc15dc4cf589c9f90712252574 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574 2024-11-07T15:54:50,555 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T15:54:50,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/fed1ee84284548a1baef5dfb5f720f1f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f 2024-11-07T15:54:50,559 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f, entries=150, sequenceid=41, filesize=11.7 K 2024-11-07T15:54:50,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 43137c94133f8cabe05776420e1d74bb in 1668ms, sequenceid=41, compaction requested=false 2024-11-07T15:54:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-07T15:54:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-07T15:54:50,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-07T15:54:50,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5100 sec 2024-11-07T15:54:50,566 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.5160 sec 2024-11-07T15:54:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-07T15:54:51,204 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:51,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-07T15:54:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:51,206 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:51,207 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:51,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:51,319 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:51,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:54:51,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078644be73dc6a41a88c11a4f768762634_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:51,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742160_1336 (size=17034) 2024-11-07T15:54:51,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:51,339 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078644be73dc6a41a88c11a4f768762634_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078644be73dc6a41a88c11a4f768762634_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:51,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/0a209971da9943729aee9727a090a938, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:51,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/0a209971da9943729aee9727a090a938 is 175, key is test_row_0/A:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:51,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742161_1337 (size=48139) 2024-11-07T15:54:51,347 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/0a209971da9943729aee9727a090a938 2024-11-07T15:54:51,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fc805e2434ac4c118746368d264971dd is 50, key is test_row_0/B:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:51,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742162_1338 (size=12001) 2024-11-07T15:54:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994951425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994951439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994951439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994951473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994951473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,481 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,482 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:51,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:51,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994951543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994951563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994951563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fc805e2434ac4c118746368d264971dd 2024-11-07T15:54:51,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/0b2ceb185d8042098e01aecc2a761d1d is 50, key is test_row_0/C:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:51,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994951763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742163_1339 (size=12001) 2024-11-07T15:54:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994951783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994951783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:51,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:51,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:51,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:51,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:52,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994952083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994952103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994952103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:52,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:52,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/0b2ceb185d8042098e01aecc2a761d1d 2024-11-07T15:54:52,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/0a209971da9943729aee9727a090a938 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938 2024-11-07T15:54:52,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938, entries=250, sequenceid=55, filesize=47.0 K 2024-11-07T15:54:52,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fc805e2434ac4c118746368d264971dd as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd 2024-11-07T15:54:52,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:54:52,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/0b2ceb185d8042098e01aecc2a761d1d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d 2024-11-07T15:54:52,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:54:52,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 43137c94133f8cabe05776420e1d74bb in 871ms, sequenceid=55, compaction requested=true 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:52,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:52,191 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:52,190 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:52,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:52,191 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:52,191 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:52,191 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:54:52,191 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:54:52,192 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,192 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,192 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.2 K 2024-11-07T15:54:52,192 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=124.3 K 2024-11-07T15:54:52,192 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,192 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938] 2024-11-07T15:54:52,192 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ae1937152e949bba1bf8bc2e7afbb0e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730994887044 2024-11-07T15:54:52,193 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 413fc05eb97c4517b9ff4f2c9a3aff2e, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730994887041 2024-11-07T15:54:52,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ec50bc15dc4cf589c9f90712252574, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994887085 2024-11-07T15:54:52,193 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf485b90f08a42a5aa35e0d9797b4c14, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994887085 2024-11-07T15:54:52,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fc805e2434ac4c118746368d264971dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:52,193 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a209971da9943729aee9727a090a938, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:52,199 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:52,201 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107e9e9a087f9524edc9966d3052c1d364d_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:52,201 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:52,201 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/a36eafc12a124cf8b13452e8dd14d88c is 50, key is test_row_0/B:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:52,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107e9e9a087f9524edc9966d3052c1d364d_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:52,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e9e9a087f9524edc9966d3052c1d364d_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:52,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742165_1341 (size=12104) 2024-11-07T15:54:52,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742164_1340 (size=4469) 2024-11-07T15:54:52,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-07T15:54:52,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,278 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:54:52,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d3137bc5d34f43a5883354706eb910d3_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994891437/Put/seqid=0 2024-11-07T15:54:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742166_1342 (size=12154) 2024-11-07T15:54:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:52,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:52,618 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:52,619 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a092080df9c6499e914cd4fc0efc857d is 175, key is test_row_0/A:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:52,621 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/a36eafc12a124cf8b13452e8dd14d88c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/a36eafc12a124cf8b13452e8dd14d88c 2024-11-07T15:54:52,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742167_1343 (size=31058) 2024-11-07T15:54:52,629 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into a36eafc12a124cf8b13452e8dd14d88c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:52,629 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:52,629 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=13, startTime=1730994892190; duration=0sec 2024-11-07T15:54:52,630 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:52,630 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:54:52,630 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:52,631 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:52,631 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a092080df9c6499e914cd4fc0efc857d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d 2024-11-07T15:54:52,631 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:54:52,631 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:52,631 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.2 K 2024-11-07T15:54:52,632 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 042fdac84e0a4d1e9c45313b0a731e23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1730994887044 2024-11-07T15:54:52,632 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fed1ee84284548a1baef5dfb5f720f1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1730994887085 2024-11-07T15:54:52,633 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2ceb185d8042098e01aecc2a761d1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:52,636 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into a092080df9c6499e914cd4fc0efc857d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:52,636 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:52,636 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=13, startTime=1730994892190; duration=0sec 2024-11-07T15:54:52,636 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:52,637 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:54:52,643 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:52,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/f736c9d3e72544b9b80bd83a49f43bad is 50, key is test_row_0/C:col10/1730994891319/Put/seqid=0 2024-11-07T15:54:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742168_1344 (size=12104) 2024-11-07T15:54:52,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994952638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,654 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/f736c9d3e72544b9b80bd83a49f43bad as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f736c9d3e72544b9b80bd83a49f43bad 2024-11-07T15:54:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994952647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994952649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,659 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into f736c9d3e72544b9b80bd83a49f43bad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:52,659 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:52,659 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=13, startTime=1730994892190; duration=0sec 2024-11-07T15:54:52,659 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:52,659 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:52,697 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d3137bc5d34f43a5883354706eb910d3_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3137bc5d34f43a5883354706eb910d3_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:52,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/42088dc653ec4fb5a4d3b7f7dd200d1d, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/42088dc653ec4fb5a4d3b7f7dd200d1d is 175, key is test_row_0/A:col10/1730994891437/Put/seqid=0 2024-11-07T15:54:52,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742169_1345 (size=30955) 2024-11-07T15:54:52,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/42088dc653ec4fb5a4d3b7f7dd200d1d 2024-11-07T15:54:52,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fccecb7721c84a03ad14e73d487d9e55 is 50, key is test_row_0/B:col10/1730994891437/Put/seqid=0 2024-11-07T15:54:52,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742170_1346 (size=12001) 2024-11-07T15:54:52,713 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fccecb7721c84a03ad14e73d487d9e55 2024-11-07T15:54:52,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/86dff72e38644054b32934074e7987a4 is 50, key is test_row_0/C:col10/1730994891437/Put/seqid=0 2024-11-07T15:54:52,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742171_1347 (size=12001) 2024-11-07T15:54:52,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994952753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994952763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994952763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994952962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994952973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994952973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:52,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:52,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,126 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/86dff72e38644054b32934074e7987a4 2024-11-07T15:54:53,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/42088dc653ec4fb5a4d3b7f7dd200d1d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d 2024-11-07T15:54:53,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d, entries=150, sequenceid=78, filesize=30.2 K 2024-11-07T15:54:53,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/fccecb7721c84a03ad14e73d487d9e55 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55 2024-11-07T15:54:53,139 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T15:54:53,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/86dff72e38644054b32934074e7987a4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4 2024-11-07T15:54:53,145 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-07T15:54:53,147 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 43137c94133f8cabe05776420e1d74bb in 868ms, sequenceid=78, compaction requested=false 2024-11-07T15:54:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-07T15:54:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-07T15:54:53,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-07T15:54:53,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-07T15:54:53,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9470 sec 2024-11-07T15:54:53,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:53,276 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:53,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:54:53,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071a963631517049a19be3723759848a50_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742172_1348 (size=14594) 2024-11-07T15:54:53,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-07T15:54:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994953339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,354 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-07T15:54:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:53,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:53,356 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:53,357 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:53,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994953347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994953353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994953463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994953463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,473 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994953473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994953493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,497 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994953503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,508 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:53,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:53,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:53,689 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:53,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994953683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994953683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411071a963631517049a19be3723759848a50_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071a963631517049a19be3723759848a50_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/e8322df2e6a24c4a9a9497be18293904, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:53,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/e8322df2e6a24c4a9a9497be18293904 is 175, key is test_row_0/A:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:53,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742173_1349 (size=39549) 2024-11-07T15:54:53,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:53,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994953693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:53,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:53,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:53,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:53,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:54,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994954003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994954003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994954013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/e8322df2e6a24c4a9a9497be18293904 2024-11-07T15:54:54,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/06a324184bb44f22b20c7763c52897f5 is 50, key is test_row_0/B:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742174_1350 (size=12001) 2024-11-07T15:54:54,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:54,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/06a324184bb44f22b20c7763c52897f5 2024-11-07T15:54:54,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e616beadc8414c5dae603d8ac5a92ba1 is 50, key is test_row_0/C:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:54,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994954513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742175_1351 (size=12001) 2024-11-07T15:54:54,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994954523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994954524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:54,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:54,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:54,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e616beadc8414c5dae603d8ac5a92ba1 2024-11-07T15:54:54,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/e8322df2e6a24c4a9a9497be18293904 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904 2024-11-07T15:54:54,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904, entries=200, sequenceid=95, filesize=38.6 K 2024-11-07T15:54:54,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/06a324184bb44f22b20c7763c52897f5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5 2024-11-07T15:54:54,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T15:54:54,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e616beadc8414c5dae603d8ac5a92ba1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1 2024-11-07T15:54:54,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1, entries=150, sequenceid=95, filesize=11.7 K 2024-11-07T15:54:54,941 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 43137c94133f8cabe05776420e1d74bb in 1665ms, sequenceid=95, compaction requested=true 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:54,941 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:54,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:54,941 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:54,942 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:54,942 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:54,942 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:54:54,942 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:54:54,942 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,942 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,942 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=99.2 K 2024-11-07T15:54:54,942 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/a36eafc12a124cf8b13452e8dd14d88c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.3 K 2024-11-07T15:54:54,942 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:54,942 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904] 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting a36eafc12a124cf8b13452e8dd14d88c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a092080df9c6499e914cd4fc0efc857d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42088dc653ec4fb5a4d3b7f7dd200d1d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730994891405 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fccecb7721c84a03ad14e73d487d9e55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730994891405 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8322df2e6a24c4a9a9497be18293904, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:54,943 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a324184bb44f22b20c7763c52897f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:54,949 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:54,952 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:54,952 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/845c822649004b60b59dede25b27434e is 50, key is test_row_0/B:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:54,954 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411075a576fd7589c477a9bc2e53a62fdb68d_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:54,956 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411075a576fd7589c477a9bc2e53a62fdb68d_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:54,956 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075a576fd7589c477a9bc2e53a62fdb68d_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:54,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742177_1353 (size=4469) 2024-11-07T15:54:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742176_1352 (size=12207) 2024-11-07T15:54:55,000 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/845c822649004b60b59dede25b27434e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/845c822649004b60b59dede25b27434e 2024-11-07T15:54:55,010 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into 845c822649004b60b59dede25b27434e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:55,010 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:55,011 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=13, startTime=1730994894941; duration=0sec 2024-11-07T15:54:55,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:55,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:54:55,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:55,012 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:55,012 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:54:55,012 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:55,012 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f736c9d3e72544b9b80bd83a49f43bad, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.3 K 2024-11-07T15:54:55,013 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting f736c9d3e72544b9b80bd83a49f43bad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994889282 2024-11-07T15:54:55,013 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 86dff72e38644054b32934074e7987a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1730994891405 2024-11-07T15:54:55,013 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e616beadc8414c5dae603d8ac5a92ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:55,021 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#296 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:55,021 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/0690c0d5a8dd4865af77783a2ab182be is 50, key is test_row_0/C:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:55,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742178_1354 (size=12207) 2024-11-07T15:54:55,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-07T15:54:55,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:55,041 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:54:55,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a0092f38342043dc87244aba04fe814f_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994893322/Put/seqid=0 2024-11-07T15:54:55,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742179_1355 (size=12154) 2024-11-07T15:54:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:55,057 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a0092f38342043dc87244aba04fe814f_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a0092f38342043dc87244aba04fe814f_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:55,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a06bdf1ca3964b41990a4ff18d9ac316, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:55,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a06bdf1ca3964b41990a4ff18d9ac316 is 175, key is test_row_0/A:col10/1730994893322/Put/seqid=0 2024-11-07T15:54:55,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742180_1356 (size=30955) 2024-11-07T15:54:55,064 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a06bdf1ca3964b41990a4ff18d9ac316 2024-11-07T15:54:55,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5e958470e94e31915f0ef83c52124e is 50, key is test_row_0/B:col10/1730994893322/Put/seqid=0 2024-11-07T15:54:55,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742181_1357 (size=12001) 2024-11-07T15:54:55,383 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#294 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:55,383 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/50c67aa25fa447b28d0d7ce57558ebe0 is 175, key is test_row_0/A:col10/1730994893275/Put/seqid=0 2024-11-07T15:54:55,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742182_1358 (size=31161) 2024-11-07T15:54:55,438 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/0690c0d5a8dd4865af77783a2ab182be as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0690c0d5a8dd4865af77783a2ab182be 2024-11-07T15:54:55,443 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into 0690c0d5a8dd4865af77783a2ab182be(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:55,443 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:55,443 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=13, startTime=1730994894941; duration=0sec 2024-11-07T15:54:55,443 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:55,443 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:54:55,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5e958470e94e31915f0ef83c52124e 2024-11-07T15:54:55,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2f76955b7b114f3eae4f194308faa869 is 50, key is test_row_0/C:col10/1730994893322/Put/seqid=0 2024-11-07T15:54:55,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742183_1359 (size=12001) 2024-11-07T15:54:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:55,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:55,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994955574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994955576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994955576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994955693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994955693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994955693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,792 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/50c67aa25fa447b28d0d7ce57558ebe0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0 2024-11-07T15:54:55,796 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into 50c67aa25fa447b28d0d7ce57558ebe0(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:55,796 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:55,796 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=13, startTime=1730994894941; duration=0sec 2024-11-07T15:54:55,796 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:55,796 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:54:55,887 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2f76955b7b114f3eae4f194308faa869 2024-11-07T15:54:55,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/a06bdf1ca3964b41990a4ff18d9ac316 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316 2024-11-07T15:54:55,894 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316, entries=150, sequenceid=118, filesize=30.2 K 2024-11-07T15:54:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5e958470e94e31915f0ef83c52124e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e 2024-11-07T15:54:55,899 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T15:54:55,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2f76955b7b114f3eae4f194308faa869 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869 2024-11-07T15:54:55,902 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869, entries=150, sequenceid=118, filesize=11.7 K 2024-11-07T15:54:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 43137c94133f8cabe05776420e1d74bb in 862ms, sequenceid=118, compaction requested=false 2024-11-07T15:54:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-07T15:54:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-07T15:54:55,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-07T15:54:55,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5470 sec 2024-11-07T15:54:55,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.5530 sec 2024-11-07T15:54:55,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:55,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:55,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:54:55,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6a30977b3664363b7e50241dab79de5_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742184_1360 (size=12304) 2024-11-07T15:54:55,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994955968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994955977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:55,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994955980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:55,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994956082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994956083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994956093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994956292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994956303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994956303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,311 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,332 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:56,336 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e6a30977b3664363b7e50241dab79de5_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6a30977b3664363b7e50241dab79de5_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:56,337 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/df12cb867c1348c4b654c6e579f19476, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:56,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/df12cb867c1348c4b654c6e579f19476 is 175, key is test_row_0/A:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:56,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742185_1361 (size=31105) 2024-11-07T15:54:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994956603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994956613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:56,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994956613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:56,621 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:56,747 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/df12cb867c1348c4b654c6e579f19476 2024-11-07T15:54:56,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/162f07061e6c47fd96badbdd9539ed3c is 50, key is test_row_0/B:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:56,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742186_1362 (size=12151) 2024-11-07T15:54:56,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/162f07061e6c47fd96badbdd9539ed3c 2024-11-07T15:54:56,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/cacd327725164dc6acfabf939670abc6 is 50, key is test_row_0/C:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:56,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742187_1363 (size=12151) 2024-11-07T15:54:57,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994957113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994957133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994957133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/cacd327725164dc6acfabf939670abc6 2024-11-07T15:54:57,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/df12cb867c1348c4b654c6e579f19476 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476 2024-11-07T15:54:57,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476, entries=150, sequenceid=136, filesize=30.4 K 2024-11-07T15:54:57,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/162f07061e6c47fd96badbdd9539ed3c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c 2024-11-07T15:54:57,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c, entries=150, sequenceid=136, filesize=11.9 K 2024-11-07T15:54:57,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/cacd327725164dc6acfabf939670abc6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6 2024-11-07T15:54:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6, entries=150, sequenceid=136, filesize=11.9 K 2024-11-07T15:54:57,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 43137c94133f8cabe05776420e1d74bb in 1281ms, sequenceid=136, compaction requested=true 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:57,200 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:54:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:57,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:57,201 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:57,201 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:57,201 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:54:57,201 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:54:57,201 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,201 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,202 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=91.0 K 2024-11-07T15:54:57,202 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/845c822649004b60b59dede25b27434e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.5 K 2024-11-07T15:54:57,202 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476] 2024-11-07T15:54:57,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50c67aa25fa447b28d0d7ce57558ebe0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:57,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 845c822649004b60b59dede25b27434e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:57,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5e958470e94e31915f0ef83c52124e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730994893322 2024-11-07T15:54:57,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a06bdf1ca3964b41990a4ff18d9ac316, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730994893322 2024-11-07T15:54:57,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 162f07061e6c47fd96badbdd9539ed3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:54:57,203 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting df12cb867c1348c4b654c6e579f19476, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:54:57,209 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:57,209 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/8175e48de367461596b877da032c9a44 is 50, key is test_row_0/B:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:57,210 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:57,212 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107cf7126c2c05f445aa2c829ad8381abd0_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:57,214 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107cf7126c2c05f445aa2c829ad8381abd0_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:57,214 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cf7126c2c05f445aa2c829ad8381abd0_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:57,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742188_1364 (size=12459) 2024-11-07T15:54:57,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742189_1365 (size=4469) 2024-11-07T15:54:57,220 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/8175e48de367461596b877da032c9a44 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8175e48de367461596b877da032c9a44 2024-11-07T15:54:57,224 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into 8175e48de367461596b877da032c9a44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:57,224 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:57,224 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=13, startTime=1730994897200; duration=0sec 2024-11-07T15:54:57,224 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:54:57,224 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:54:57,224 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:54:57,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:54:57,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:54:57,225 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,225 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0690c0d5a8dd4865af77783a2ab182be, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=35.5 K 2024-11-07T15:54:57,226 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 0690c0d5a8dd4865af77783a2ab182be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1730994892612 2024-11-07T15:54:57,226 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f76955b7b114f3eae4f194308faa869, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1730994893322 2024-11-07T15:54:57,226 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting cacd327725164dc6acfabf939670abc6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:54:57,231 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:57,231 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e0213032970449eabae4fd079c5d9005 is 50, key is test_row_0/C:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:57,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742190_1366 (size=12459) 2024-11-07T15:54:57,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-07T15:54:57,513 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:54:57,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:54:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-07T15:54:57,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:57,516 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:54:57,517 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:54:57,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:54:57,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:57,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:54:57,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-07T15:54:57,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076cdc79f8792a4abdb41f5ab3eb678302_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994895975/Put/seqid=0 2024-11-07T15:54:57,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742191_1367 (size=14794) 2024-11-07T15:54:57,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994957582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994957583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,620 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:54:57,620 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/38e031e723504498b65b46f21988f1be is 175, key is test_row_0/A:col10/1730994895919/Put/seqid=0 2024-11-07T15:54:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:57,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742192_1368 (size=31413) 2024-11-07T15:54:57,628 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/38e031e723504498b65b46f21988f1be as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be 2024-11-07T15:54:57,632 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into 38e031e723504498b65b46f21988f1be(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:57,632 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:57,632 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=13, startTime=1730994897200; duration=0sec 2024-11-07T15:54:57,632 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:57,632 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:54:57,638 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e0213032970449eabae4fd079c5d9005 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e0213032970449eabae4fd079c5d9005 2024-11-07T15:54:57,642 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into e0213032970449eabae4fd079c5d9005(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:54:57,642 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:57,642 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=13, startTime=1730994897200; duration=0sec 2024-11-07T15:54:57,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:54:57,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:54:57,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:57,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:57,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994957693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994957703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994957903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:57,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994957923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:57,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:57,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:57,937 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076cdc79f8792a4abdb41f5ab3eb678302_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076cdc79f8792a4abdb41f5ab3eb678302_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:57,938 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/011b4f62cb494f979c91a0a99e58c6ca, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:57,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/011b4f62cb494f979c91a0a99e58c6ca is 175, key is test_row_0/A:col10/1730994895975/Put/seqid=0 2024-11-07T15:54:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742193_1369 (size=39749) 2024-11-07T15:54:57,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/011b4f62cb494f979c91a0a99e58c6ca 2024-11-07T15:54:57,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5c2eaf7ed4410195540bdbb59a1e65 is 50, key is test_row_0/B:col10/1730994895975/Put/seqid=0 2024-11-07T15:54:57,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742194_1370 (size=12151) 2024-11-07T15:54:57,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:57,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:58,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:58,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994958143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,149 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:58,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994958153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994958153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,161 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,162 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 43137c94133f8cabe05776420e1d74bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994958213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994958242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5c2eaf7ed4410195540bdbb59a1e65 2024-11-07T15:54:58,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afcdc1918b0d4f93be8bcb97e72169b8 is 50, key is test_row_0/C:col10/1730994895975/Put/seqid=0 2024-11-07T15:54:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742195_1371 (size=12151) 2024-11-07T15:54:58,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:58,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994958723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:58,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:54:58,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afcdc1918b0d4f93be8bcb97e72169b8 2024-11-07T15:54:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994958763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:58,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/011b4f62cb494f979c91a0a99e58c6ca as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca 2024-11-07T15:54:58,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca, entries=200, sequenceid=158, filesize=38.8 K 2024-11-07T15:54:58,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/ec5c2eaf7ed4410195540bdbb59a1e65 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65 2024-11-07T15:54:58,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T15:54:58,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afcdc1918b0d4f93be8bcb97e72169b8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8 2024-11-07T15:54:58,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8, entries=150, sequenceid=158, filesize=11.9 K 2024-11-07T15:54:58,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 43137c94133f8cabe05776420e1d74bb in 1267ms, sequenceid=158, compaction requested=false 2024-11-07T15:54:58,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:54:58,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-07T15:54:58,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:54:58,893 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:54:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072b8c199332b14739a097308ba8f80ada_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994897581/Put/seqid=0 2024-11-07T15:54:58,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742196_1372 (size=12304) 2024-11-07T15:54:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:54:58,910 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072b8c199332b14739a097308ba8f80ada_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072b8c199332b14739a097308ba8f80ada_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:58,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/7ac2b3d86d1143fc969123b28b3ca35b, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:54:58,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/7ac2b3d86d1143fc969123b28b3ca35b is 175, key is test_row_0/A:col10/1730994897581/Put/seqid=0 2024-11-07T15:54:58,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742197_1373 (size=31105) 2024-11-07T15:54:59,319 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/7ac2b3d86d1143fc969123b28b3ca35b 2024-11-07T15:54:59,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/57a5af3432b548e6931a7396fa91eea1 is 50, key is test_row_0/B:col10/1730994897581/Put/seqid=0 2024-11-07T15:54:59,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742198_1374 (size=12151) 2024-11-07T15:54:59,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:54:59,733 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/57a5af3432b548e6931a7396fa91eea1 2024-11-07T15:54:59,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e598bb36abe6421ca71cad029a48b4b4 is 50, key is test_row_0/C:col10/1730994897581/Put/seqid=0 2024-11-07T15:54:59,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742199_1375 (size=12151) 2024-11-07T15:54:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:54:59,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:54:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994959842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:59,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994959844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:59,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994959963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:59,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:54:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:54:59,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994959963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:54:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:54:59,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,145 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e598bb36abe6421ca71cad029a48b4b4 2024-11-07T15:55:00,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/7ac2b3d86d1143fc969123b28b3ca35b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b 2024-11-07T15:55:00,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b, entries=150, sequenceid=175, filesize=30.4 K 2024-11-07T15:55:00,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/57a5af3432b548e6931a7396fa91eea1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1 2024-11-07T15:55:00,156 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1, entries=150, sequenceid=175, filesize=11.9 K 2024-11-07T15:55:00,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/e598bb36abe6421ca71cad029a48b4b4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4 2024-11-07T15:55:00,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4, entries=150, sequenceid=175, filesize=11.9 K 2024-11-07T15:55:00,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 43137c94133f8cabe05776420e1d74bb in 1269ms, sequenceid=175, compaction requested=true 2024-11-07T15:55:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:00,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-07T15:55:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-07T15:55:00,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-07T15:55:00,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-07T15:55:00,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.6510 sec 2024-11-07T15:55:00,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:00,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:00,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:55:00,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110736883de2fa7e42f69c1eeebf38e15951_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:00,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742200_1376 (size=14794) 2024-11-07T15:55:00,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994960211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994960214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994960215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994960217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994960217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994960333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994960333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994960334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,344 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994960343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994960343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994960543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994960543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994960552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994960553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994960553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,588 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:00,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110736883de2fa7e42f69c1eeebf38e15951_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736883de2fa7e42f69c1eeebf38e15951_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:00,593 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/9cddf2758df148e2b33f0d0f12899f31, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:00,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/9cddf2758df148e2b33f0d0f12899f31 is 175, key is test_row_0/A:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:00,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742201_1377 (size=39749) 2024-11-07T15:55:00,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994960863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994960863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994960872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994960873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994960873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:00,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/9cddf2758df148e2b33f0d0f12899f31 2024-11-07T15:55:01,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/32699106ca65400d9f62c9f6282cbce1 is 50, key is test_row_0/B:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:01,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742202_1378 (size=12151) 2024-11-07T15:55:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994961383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:01,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994961383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:01,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994961384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:01,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994961384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:01,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:01,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994961393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:01,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:01,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/32699106ca65400d9f62c9f6282cbce1 2024-11-07T15:55:01,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/c032d94cc3ae4a37aec23fa4e838b44d is 50, key is test_row_0/C:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:01,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742203_1379 (size=12151) 2024-11-07T15:55:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-07T15:55:01,673 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:01,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:01,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-07T15:55:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:55:01,676 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:01,676 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:01,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:01,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:55:01,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/c032d94cc3ae4a37aec23fa4e838b44d 2024-11-07T15:55:01,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/9cddf2758df148e2b33f0d0f12899f31 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31 2024-11-07T15:55:01,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31, entries=200, sequenceid=197, filesize=38.8 K 2024-11-07T15:55:01,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T15:55:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:01,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:01,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/32699106ca65400d9f62c9f6282cbce1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1 2024-11-07T15:55:01,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1, entries=150, sequenceid=197, filesize=11.9 K 2024-11-07T15:55:01,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/c032d94cc3ae4a37aec23fa4e838b44d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d 2024-11-07T15:55:01,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d, entries=150, sequenceid=197, filesize=11.9 K 2024-11-07T15:55:01,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 43137c94133f8cabe05776420e1d74bb in 1671ms, sequenceid=197, compaction requested=true 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:01,841 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:01,841 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:01,842 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142016 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:01,842 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:01,842 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:55:01,842 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:55:01,843 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,843 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,843 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=138.7 K 2024-11-07T15:55:01,843 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8175e48de367461596b877da032c9a44, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=47.8 K 2024-11-07T15:55:01,843 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,843 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31] 2024-11-07T15:55:01,843 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8175e48de367461596b877da032c9a44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:55:01,843 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38e031e723504498b65b46f21988f1be, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:55:01,843 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 011b4f62cb494f979c91a0a99e58c6ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730994895961 2024-11-07T15:55:01,843 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ec5c2eaf7ed4410195540bdbb59a1e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730994895975 2024-11-07T15:55:01,844 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ac2b3d86d1143fc969123b28b3ca35b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730994897567 2024-11-07T15:55:01,844 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a5af3432b548e6931a7396fa91eea1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730994897567 2024-11-07T15:55:01,844 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cddf2758df148e2b33f0d0f12899f31, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:01,844 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 32699106ca65400d9f62c9f6282cbce1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:01,850 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:01,851 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107b7e5c8c5bd3e4065bca8fbfeb88bc53a_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:01,852 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:01,852 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/b5063f46fc29479d97a57b34433c1e3e is 50, key is test_row_0/B:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:01,854 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107b7e5c8c5bd3e4065bca8fbfeb88bc53a_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:01,854 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107b7e5c8c5bd3e4065bca8fbfeb88bc53a_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:01,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742204_1380 (size=12595) 2024-11-07T15:55:01,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742205_1381 (size=4469) 2024-11-07T15:55:01,862 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/b5063f46fc29479d97a57b34433c1e3e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b5063f46fc29479d97a57b34433c1e3e 2024-11-07T15:55:01,867 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into b5063f46fc29479d97a57b34433c1e3e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:01,867 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=12, startTime=1730994901841; duration=0sec 2024-11-07T15:55:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:55:01,867 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:01,868 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:01,868 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:55:01,868 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,868 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e0213032970449eabae4fd079c5d9005, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=47.8 K 2024-11-07T15:55:01,868 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e0213032970449eabae4fd079c5d9005, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1730994895573 2024-11-07T15:55:01,868 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting afcdc1918b0d4f93be8bcb97e72169b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1730994895975 2024-11-07T15:55:01,869 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e598bb36abe6421ca71cad029a48b4b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1730994897567 2024-11-07T15:55:01,869 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c032d94cc3ae4a37aec23fa4e838b44d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:01,876 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:01,876 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/424e35afcffe415ba0b74d932b36cd8a is 50, key is test_row_0/C:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:01,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742206_1382 (size=12595) 2024-11-07T15:55:01,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-07T15:55:01,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:01,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:55:01,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110718654680561e4faeb2e552fa3f51339b_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994900212/Put/seqid=0 2024-11-07T15:55:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:55:01,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742207_1383 (size=12304) 2024-11-07T15:55:02,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:02,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110718654680561e4faeb2e552fa3f51339b_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718654680561e4faeb2e552fa3f51339b_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/f276c1f03eb442a8afd7302bd89a6740, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:02,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/f276c1f03eb442a8afd7302bd89a6740 is 175, key is test_row_0/A:col10/1730994900212/Put/seqid=0 2024-11-07T15:55:02,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742208_1384 (size=31105) 2024-11-07T15:55:02,014 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/f276c1f03eb442a8afd7302bd89a6740 2024-11-07T15:55:02,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/462676543be2438ba1b9e8840c3d49ba is 50, key is test_row_0/B:col10/1730994900212/Put/seqid=0 2024-11-07T15:55:02,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742209_1385 (size=12151) 2024-11-07T15:55:02,027 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/462676543be2438ba1b9e8840c3d49ba 2024-11-07T15:55:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/55a3a255a22643feb38ed64dc9edf4f3 is 50, key is test_row_0/C:col10/1730994900212/Put/seqid=0 2024-11-07T15:55:02,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742210_1386 (size=12151) 2024-11-07T15:55:02,263 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:02,263 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6bf34f2cb92c46189b005630f907a25b is 175, key is test_row_0/A:col10/1730994899843/Put/seqid=0 2024-11-07T15:55:02,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742211_1387 (size=31549) 2024-11-07T15:55:02,284 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/424e35afcffe415ba0b74d932b36cd8a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/424e35afcffe415ba0b74d932b36cd8a 2024-11-07T15:55:02,288 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into 424e35afcffe415ba0b74d932b36cd8a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:02,288 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:02,288 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=12, startTime=1730994901841; duration=0sec 2024-11-07T15:55:02,288 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:02,288 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:55:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:55:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:02,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:02,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/55a3a255a22643feb38ed64dc9edf4f3 2024-11-07T15:55:02,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/f276c1f03eb442a8afd7302bd89a6740 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740 2024-11-07T15:55:02,449 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740, entries=150, sequenceid=213, filesize=30.4 K 2024-11-07T15:55:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/462676543be2438ba1b9e8840c3d49ba as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba 2024-11-07T15:55:02,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994962438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,453 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T15:55:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994962439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994962440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/55a3a255a22643feb38ed64dc9edf4f3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T15:55:02,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 43137c94133f8cabe05776420e1d74bb in 479ms, sequenceid=213, compaction requested=false 2024-11-07T15:55:02,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:02,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:02,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-07T15:55:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-07T15:55:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:02,464 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:02,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:55:02,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-07T15:55:02,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 787 msec 2024-11-07T15:55:02,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 791 msec 2024-11-07T15:55:02,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ca736b0ccbb476bb51765f4111e4665_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:02,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742212_1388 (size=14794) 2024-11-07T15:55:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994962510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994962519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994962563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994962563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994962563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994962632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994962633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,676 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6bf34f2cb92c46189b005630f907a25b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b 2024-11-07T15:55:02,680 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into 6bf34f2cb92c46189b005630f907a25b(size=30.8 K), total size for store is 61.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:02,680 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:02,680 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=12, startTime=1730994901841; duration=0sec 2024-11-07T15:55:02,680 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:02,680 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:55:02,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994962773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994962773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994962783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-07T15:55:02,814 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:02,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-07T15:55:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:02,816 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:02,817 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:02,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994962853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994962853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:02,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:02,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:02,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411072ca736b0ccbb476bb51765f4111e4665_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ca736b0ccbb476bb51765f4111e4665_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:02,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b96fc74bd9ac4a63865a6c127959142f, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:02,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b96fc74bd9ac4a63865a6c127959142f is 175, key is test_row_0/A:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:02,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742213_1389 (size=39749) 2024-11-07T15:55:02,895 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b96fc74bd9ac4a63865a6c127959142f 2024-11-07T15:55:02,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/66e1edabcfc847f5bd2caa6461d9613a is 50, key is test_row_0/B:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:02,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742214_1390 (size=12151) 2024-11-07T15:55:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:02,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:02,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:02,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:02,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:02,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:02,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994963093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994963093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994963103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:03,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994963173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994963173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:03,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/66e1edabcfc847f5bd2caa6461d9613a 2024-11-07T15:55:03,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/04ef060d9fba4798a4a40329960c5e08 is 50, key is test_row_0/C:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:03,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742215_1391 (size=12151) 2024-11-07T15:55:03,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:03,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994963613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994963613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994963623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994963693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994963693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:03,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/04ef060d9fba4798a4a40329960c5e08 2024-11-07T15:55:03,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b96fc74bd9ac4a63865a6c127959142f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f 2024-11-07T15:55:03,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f, entries=200, sequenceid=236, filesize=38.8 K 2024-11-07T15:55:03,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/66e1edabcfc847f5bd2caa6461d9613a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a 2024-11-07T15:55:03,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T15:55:03,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/04ef060d9fba4798a4a40329960c5e08 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08 2024-11-07T15:55:03,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:03,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08, entries=150, sequenceid=236, filesize=11.9 K 2024-11-07T15:55:03,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 43137c94133f8cabe05776420e1d74bb in 1275ms, sequenceid=236, compaction requested=true 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:03,739 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:03,739 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:03,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:03,740 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:03,740 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:03,740 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:55:03,740 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:55:03,740 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,740 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,741 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b5063f46fc29479d97a57b34433c1e3e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=36.0 K 2024-11-07T15:55:03,741 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=100.0 K 2024-11-07T15:55:03,741 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,741 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f] 2024-11-07T15:55:03,741 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b5063f46fc29479d97a57b34433c1e3e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:03,741 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bf34f2cb92c46189b005630f907a25b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:03,741 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 462676543be2438ba1b9e8840c3d49ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994900212 2024-11-07T15:55:03,741 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting f276c1f03eb442a8afd7302bd89a6740, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994900212 2024-11-07T15:55:03,742 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e1edabcfc847f5bd2caa6461d9613a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902438 2024-11-07T15:55:03,742 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b96fc74bd9ac4a63865a6c127959142f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902435 2024-11-07T15:55:03,748 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:03,750 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:03,750 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/b7b8c6d0138c4672a08891c3cb44b55c is 50, key is test_row_0/B:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:03,752 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107a374e6084509417f957b164fa7952f7b_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:03,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742216_1392 (size=12697) 2024-11-07T15:55:03,754 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107a374e6084509417f957b164fa7952f7b_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:03,754 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107a374e6084509417f957b164fa7952f7b_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:03,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742217_1393 (size=4469) 2024-11-07T15:55:03,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-07T15:55:03,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:03,888 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:55:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ceac58fce2e466abe679d7818ce531c_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994902501/Put/seqid=0 2024-11-07T15:55:03,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742218_1394 (size=12304) 2024-11-07T15:55:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:04,174 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:04,174 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/05049a21d27d45be82c485ce93618733 is 175, key is test_row_0/A:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:04,180 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/b7b8c6d0138c4672a08891c3cb44b55c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b7b8c6d0138c4672a08891c3cb44b55c 2024-11-07T15:55:04,185 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into b7b8c6d0138c4672a08891c3cb44b55c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:04,185 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:04,185 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=13, startTime=1730994903739; duration=0sec 2024-11-07T15:55:04,185 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:04,185 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:55:04,185 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:04,186 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:04,186 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:55:04,186 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:04,187 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/424e35afcffe415ba0b74d932b36cd8a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=36.0 K 2024-11-07T15:55:04,187 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 424e35afcffe415ba0b74d932b36cd8a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1730994899815 2024-11-07T15:55:04,187 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a3a255a22643feb38ed64dc9edf4f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994900212 2024-11-07T15:55:04,187 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 04ef060d9fba4798a4a40329960c5e08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902438 2024-11-07T15:55:04,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742219_1395 (size=31651) 2024-11-07T15:55:04,201 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:04,201 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/8777981f017d43808531985cfa97c189 is 50, key is test_row_0/C:col10/1730994902463/Put/seqid=0 2024-11-07T15:55:04,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742220_1396 (size=12697) 2024-11-07T15:55:04,219 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/8777981f017d43808531985cfa97c189 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8777981f017d43808531985cfa97c189 2024-11-07T15:55:04,224 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into 8777981f017d43808531985cfa97c189(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:04,224 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:04,224 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=13, startTime=1730994903739; duration=0sec 2024-11-07T15:55:04,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:04,225 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:55:04,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,305 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411078ceac58fce2e466abe679d7818ce531c_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ceac58fce2e466abe679d7818ce531c_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:04,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/94498544d1ad4c41a77fa86c247f40fc, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:04,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/94498544d1ad4c41a77fa86c247f40fc is 175, key is test_row_0/A:col10/1730994902501/Put/seqid=0 2024-11-07T15:55:04,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742221_1397 (size=31105) 2024-11-07T15:55:04,321 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/94498544d1ad4c41a77fa86c247f40fc 2024-11-07T15:55:04,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/8c8436055584408093d329a566f13a64 is 50, key is test_row_0/B:col10/1730994902501/Put/seqid=0 2024-11-07T15:55:04,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742222_1398 (size=12151) 2024-11-07T15:55:04,352 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/8c8436055584408093d329a566f13a64 2024-11-07T15:55:04,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afd44038f201442e87e863b8e2a968d8 is 50, key is test_row_0/C:col10/1730994902501/Put/seqid=0 2024-11-07T15:55:04,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742223_1399 (size=12151) 2024-11-07T15:55:04,394 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afd44038f201442e87e863b8e2a968d8 2024-11-07T15:55:04,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/94498544d1ad4c41a77fa86c247f40fc as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc 2024-11-07T15:55:04,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc, entries=150, sequenceid=251, filesize=30.4 K 2024-11-07T15:55:04,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/8c8436055584408093d329a566f13a64 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64 2024-11-07T15:55:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:55:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/afd44038f201442e87e863b8e2a968d8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8 2024-11-07T15:55:04,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,414 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8, entries=150, sequenceid=251, filesize=11.9 K 2024-11-07T15:55:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,415 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 43137c94133f8cabe05776420e1d74bb in 526ms, sequenceid=251, compaction requested=false 2024-11-07T15:55:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-07T15:55:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-07T15:55:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-07T15:55:04,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-07T15:55:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.6060 sec 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,602 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/05049a21d27d45be82c485ce93618733 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733 2024-11-07T15:55:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,608 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into 05049a21d27d45be82c485ce93618733(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:04,609 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:04,609 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=13, startTime=1730994903739; duration=0sec 2024-11-07T15:55:04,609 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,609 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:55:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:04,753 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:04,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:55:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070ec70c9a1d42433ab4ac42936678ccbc_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742224_1400 (size=22618) 2024-11-07T15:55:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:04,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994964812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994964814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994964818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994964820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994964821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994964932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994964933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994964933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994964943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:04,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994964943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:04,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:04,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-07T15:55:04,973 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:04,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-07T15:55:04,975 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:04,976 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:04,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:05,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994965153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994965153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994965153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994965153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994965154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,170 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:05,174 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070ec70c9a1d42433ab4ac42936678ccbc_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ec70c9a1d42433ab4ac42936678ccbc_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:05,175 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6c45295ce20a4f1da178bb8b9c4c92e1, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:05,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6c45295ce20a4f1da178bb8b9c4c92e1 is 175, key is test_row_0/A:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:05,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742225_1401 (size=66023) 2024-11-07T15:55:05,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:05,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994965473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994965473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994965473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994965474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994965474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:05,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,589 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6c45295ce20a4f1da178bb8b9c4c92e1 2024-11-07T15:55:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/c5d3b0363bd6438e924cb1b5451fdb0f is 50, key is test_row_0/B:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:05,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742226_1402 (size=12301) 2024-11-07T15:55:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:05,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:05,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/c5d3b0363bd6438e924cb1b5451fdb0f 2024-11-07T15:55:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994965993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:06,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994965993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994965993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994965993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:06,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994965993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:06,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:06,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/ac248c36bf174c2e95bc51dc975c495b is 50, key is test_row_0/C:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:06,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742227_1403 (size=12301) 2024-11-07T15:55:06,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:06,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:06,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/ac248c36bf174c2e95bc51dc975c495b 2024-11-07T15:55:06,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/6c45295ce20a4f1da178bb8b9c4c92e1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1 2024-11-07T15:55:06,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1, entries=350, sequenceid=265, filesize=64.5 K 2024-11-07T15:55:06,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/c5d3b0363bd6438e924cb1b5451fdb0f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f 2024-11-07T15:55:06,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T15:55:06,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/ac248c36bf174c2e95bc51dc975c495b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b 2024-11-07T15:55:06,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b, entries=150, sequenceid=265, filesize=12.0 K 2024-11-07T15:55:06,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 43137c94133f8cabe05776420e1d74bb in 1675ms, sequenceid=265, compaction requested=true 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:06,429 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:06,429 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:06,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/A is initiating minor compaction (all files) 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/B is initiating minor compaction (all files) 2024-11-07T15:55:06,430 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/A in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,430 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/B in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,430 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=125.8 K 2024-11-07T15:55:06,430 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b7b8c6d0138c4672a08891c3cb44b55c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=36.3 K 2024-11-07T15:55:06,430 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1] 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b8c6d0138c4672a08891c3cb44b55c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902438 2024-11-07T15:55:06,430 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05049a21d27d45be82c485ce93618733, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902438 2024-11-07T15:55:06,431 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8436055584408093d329a566f13a64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994902501 2024-11-07T15:55:06,431 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94498544d1ad4c41a77fa86c247f40fc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994902501 2024-11-07T15:55:06,431 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d3b0363bd6438e924cb1b5451fdb0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730994904746 2024-11-07T15:55:06,431 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c45295ce20a4f1da178bb8b9c4c92e1, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730994904685 2024-11-07T15:55:06,438 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:06,439 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/09c5516440d148b7807ea431354ab775 is 50, key is test_row_0/B:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:06,440 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:06,442 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110735f72300abf24952b13e93997b27e890_43137c94133f8cabe05776420e1d74bb store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:06,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742228_1404 (size=12949) 2024-11-07T15:55:06,444 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110735f72300abf24952b13e93997b27e890_43137c94133f8cabe05776420e1d74bb, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:06,444 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110735f72300abf24952b13e93997b27e890_43137c94133f8cabe05776420e1d74bb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:06,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742229_1405 (size=4469) 2024-11-07T15:55:06,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-07T15:55:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,508 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-07T15:55:06,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c92a761c4ba141baa23ece3fae43bdef_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994904816/Put/seqid=0 2024-11-07T15:55:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742230_1406 (size=12454) 2024-11-07T15:55:06,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:06,523 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107c92a761c4ba141baa23ece3fae43bdef_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c92a761c4ba141baa23ece3fae43bdef_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:06,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/53445ab511864c4f813e039033c9a2e6, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:06,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/53445ab511864c4f813e039033c9a2e6 is 175, key is test_row_0/A:col10/1730994904816/Put/seqid=0 2024-11-07T15:55:06,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742231_1407 (size=31255) 2024-11-07T15:55:06,847 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/09c5516440d148b7807ea431354ab775 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/09c5516440d148b7807ea431354ab775 2024-11-07T15:55:06,849 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#A#compaction#334 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:06,850 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/3f91bfc7b8014a22b179baa3f3627a9d is 175, key is test_row_0/A:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:06,852 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/B of 43137c94133f8cabe05776420e1d74bb into 09c5516440d148b7807ea431354ab775(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:06,852 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:06,852 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/B, priority=13, startTime=1730994906429; duration=0sec 2024-11-07T15:55:06,853 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:06,853 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:55:06,853 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:06,854 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:06,855 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 43137c94133f8cabe05776420e1d74bb/C is initiating minor compaction (all files) 2024-11-07T15:55:06,855 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 43137c94133f8cabe05776420e1d74bb/C in TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:06,855 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8777981f017d43808531985cfa97c189, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp, totalSize=36.3 K 2024-11-07T15:55:06,855 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8777981f017d43808531985cfa97c189, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1730994902438 2024-11-07T15:55:06,856 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting afd44038f201442e87e863b8e2a968d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1730994902501 2024-11-07T15:55:06,856 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ac248c36bf174c2e95bc51dc975c495b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1730994904746 2024-11-07T15:55:06,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742232_1408 (size=31903) 2024-11-07T15:55:06,864 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 43137c94133f8cabe05776420e1d74bb#C#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:06,865 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/a69228b91f5b45c28d0632141ef6a441 is 50, key is test_row_0/C:col10/1730994904746/Put/seqid=0 2024-11-07T15:55:06,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742233_1409 (size=12949) 2024-11-07T15:55:06,876 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/a69228b91f5b45c28d0632141ef6a441 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/a69228b91f5b45c28d0632141ef6a441 2024-11-07T15:55:06,881 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/C of 43137c94133f8cabe05776420e1d74bb into a69228b91f5b45c28d0632141ef6a441(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:06,881 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:06,881 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/C, priority=13, startTime=1730994906429; duration=0sec 2024-11-07T15:55:06,881 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:06,881 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:55:06,930 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/53445ab511864c4f813e039033c9a2e6 2024-11-07T15:55:06,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/2a138319d5e945839a399d3c3c52fd1f is 50, key is test_row_0/B:col10/1730994904816/Put/seqid=0 2024-11-07T15:55:06,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742234_1410 (size=12301) 2024-11-07T15:55:06,943 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/2a138319d5e945839a399d3c3c52fd1f 2024-11-07T15:55:06,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/f77a8d27ea5e4f88a5f024bc5d75db14 is 50, key is test_row_0/C:col10/1730994904816/Put/seqid=0 2024-11-07T15:55:06,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742235_1411 (size=12301) 2024-11-07T15:55:07,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:07,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. as already flushing 2024-11-07T15:55:07,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994967026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994967027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994967028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994967029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994967030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,052 INFO [Thread-1509 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1509. 2024-11-07T15:55:07,052 DEBUG [Thread-1509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,052 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,052 DEBUG [Thread-1509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,058 INFO [Thread-1513 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1513. 2024-11-07T15:55:07,058 DEBUG [Thread-1513 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,058 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,058 INFO [Thread-1511 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1511. 2024-11-07T15:55:07,058 DEBUG [Thread-1511 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,058 DEBUG [Thread-1513 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,058 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,058 DEBUG [Thread-1511 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,061 INFO [Thread-1515 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1515. 2024-11-07T15:55:07,061 DEBUG [Thread-1515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,061 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,061 DEBUG [Thread-1515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,064 INFO [Thread-1507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1507. 2024-11-07T15:55:07,064 DEBUG [Thread-1507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,064 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,064 DEBUG [Thread-1507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994967143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994967143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994967143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994967143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994967143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,261 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/3f91bfc7b8014a22b179baa3f3627a9d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/3f91bfc7b8014a22b179baa3f3627a9d 2024-11-07T15:55:07,264 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 43137c94133f8cabe05776420e1d74bb/A of 43137c94133f8cabe05776420e1d74bb into 3f91bfc7b8014a22b179baa3f3627a9d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:07,264 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:07,264 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., storeName=43137c94133f8cabe05776420e1d74bb/A, priority=13, startTime=1730994906429; duration=0sec 2024-11-07T15:55:07,265 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:07,265 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:55:07,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59178 deadline: 1730994967353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59224 deadline: 1730994967353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59246 deadline: 1730994967353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/f77a8d27ea5e4f88a5f024bc5d75db14 2024-11-07T15:55:07,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59232 deadline: 1730994967353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:07,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59262 deadline: 1730994967353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:07,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:07,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/53445ab511864c4f813e039033c9a2e6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/53445ab511864c4f813e039033c9a2e6 2024-11-07T15:55:07,360 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/53445ab511864c4f813e039033c9a2e6, entries=150, sequenceid=290, filesize=30.5 K 2024-11-07T15:55:07,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/2a138319d5e945839a399d3c3c52fd1f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/2a138319d5e945839a399d3c3c52fd1f 2024-11-07T15:55:07,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/2a138319d5e945839a399d3c3c52fd1f, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T15:55:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/f77a8d27ea5e4f88a5f024bc5d75db14 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f77a8d27ea5e4f88a5f024bc5d75db14 2024-11-07T15:55:07,367 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f77a8d27ea5e4f88a5f024bc5d75db14, entries=150, sequenceid=290, filesize=12.0 K 2024-11-07T15:55:07,368 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 43137c94133f8cabe05776420e1d74bb in 860ms, sequenceid=290, compaction requested=false 2024-11-07T15:55:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:07,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-07T15:55:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-07T15:55:07,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-07T15:55:07,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3930 sec 2024-11-07T15:55:07,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.3970 sec 2024-11-07T15:55:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:07,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:07,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-07T15:55:07,664 INFO [Thread-1498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1498. 2024-11-07T15:55:07,665 DEBUG [Thread-1498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,665 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 INFO [Thread-1500 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1500. 2024-11-07T15:55:07,665 DEBUG [Thread-1500 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,665 INFO [Thread-1496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1496. 2024-11-07T15:55:07,665 DEBUG [Thread-1498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 DEBUG [Thread-1496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,665 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 DEBUG [Thread-1500 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 DEBUG [Thread-1496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,666 INFO [Thread-1502 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1502. 2024-11-07T15:55:07,666 DEBUG [Thread-1502 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,666 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,666 DEBUG [Thread-1502 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,666 INFO [Thread-1504 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1504. 2024-11-07T15:55:07,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,666 DEBUG [Thread-1504 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:07,666 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,667 DEBUG [Thread-1504 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:07,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:07,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079c6f162d328947568a58c53f05a55324_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994907663/Put/seqid=0 2024-11-07T15:55:07,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742236_1412 (size=12454) 2024-11-07T15:55:08,074 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:08,078 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079c6f162d328947568a58c53f05a55324_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079c6f162d328947568a58c53f05a55324_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:08,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/16b9ad70ed674eb4be47fa916023a5a9, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:08,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/16b9ad70ed674eb4be47fa916023a5a9 is 175, key is test_row_0/A:col10/1730994907663/Put/seqid=0 2024-11-07T15:55:08,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742237_1413 (size=31255) 2024-11-07T15:55:08,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/16b9ad70ed674eb4be47fa916023a5a9 2024-11-07T15:55:08,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/7d1b2ede5cd046cbb77188deef4c2628 is 50, key is test_row_0/B:col10/1730994907663/Put/seqid=0 2024-11-07T15:55:08,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742238_1414 (size=12301) 2024-11-07T15:55:08,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/7d1b2ede5cd046cbb77188deef4c2628 2024-11-07T15:55:08,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2c83cf9c3a9b44f7ab09e3a976cedab9 is 50, key is test_row_0/C:col10/1730994907663/Put/seqid=0 2024-11-07T15:55:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742239_1415 (size=12301) 2024-11-07T15:55:09,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-07T15:55:09,133 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1196 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3581 rows 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3591 rows 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1192 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3573 rows 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1198 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3588 rows 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1193 2024-11-07T15:55:09,133 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3576 rows 2024-11-07T15:55:09,133 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:55:09,133 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:55:09,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:09,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:09,134 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:09,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:55:09,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:09,137 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994909137"}]},"ts":"1730994909137"} 2024-11-07T15:55:09,138 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:55:09,138 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:55:09,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:55:09,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, UNASSIGN}] 2024-11-07T15:55:09,141 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, UNASSIGN 2024-11-07T15:55:09,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:09,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, UNASSIGN because future has completed 2024-11-07T15:55:09,143 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:55:09,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:09,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:09,295 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:09,295 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:55:09,295 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 43137c94133f8cabe05776420e1d74bb, disabling compactions & flushes 2024-11-07T15:55:09,295 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:09,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2c83cf9c3a9b44f7ab09e3a976cedab9 2024-11-07T15:55:09,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/16b9ad70ed674eb4be47fa916023a5a9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/16b9ad70ed674eb4be47fa916023a5a9 2024-11-07T15:55:09,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/16b9ad70ed674eb4be47fa916023a5a9, entries=150, sequenceid=307, filesize=30.5 K 2024-11-07T15:55:09,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/7d1b2ede5cd046cbb77188deef4c2628 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/7d1b2ede5cd046cbb77188deef4c2628 2024-11-07T15:55:09,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/7d1b2ede5cd046cbb77188deef4c2628, entries=150, sequenceid=307, filesize=12.0 K 2024-11-07T15:55:09,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/2c83cf9c3a9b44f7ab09e3a976cedab9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2c83cf9c3a9b44f7ab09e3a976cedab9 2024-11-07T15:55:09,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2c83cf9c3a9b44f7ab09e3a976cedab9, entries=150, sequenceid=307, filesize=12.0 K 2024-11-07T15:55:09,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=13.42 KB/13740 for 43137c94133f8cabe05776420e1d74bb in 1652ms, sequenceid=307, compaction requested=true 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 43137c94133f8cabe05776420e1d74bb: 2024-11-07T15:55:09,316 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:09,316 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:09,316 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. after waiting 0 ms 2024-11-07T15:55:09,316 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:09,316 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 43137c94133f8cabe05776420e1d74bb 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 43137c94133f8cabe05776420e1d74bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:09,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:09,316 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. because compaction request was cancelled 2024-11-07T15:55:09,316 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. because compaction request was cancelled 2024-11-07T15:55:09,316 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:B 2024-11-07T15:55:09,316 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:A 2024-11-07T15:55:09,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. because compaction request was cancelled 2024-11-07T15:55:09,317 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 43137c94133f8cabe05776420e1d74bb:C 2024-11-07T15:55:09,321 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749113f4c790b43c497675c1b6771666e_43137c94133f8cabe05776420e1d74bb is 50, key is test_row_0/A:col10/1730994907665/Put/seqid=0 2024-11-07T15:55:09,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742240_1416 (size=9914) 2024-11-07T15:55:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:09,724 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:09,728 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110749113f4c790b43c497675c1b6771666e_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749113f4c790b43c497675c1b6771666e_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:09,729 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b920a5596912458c809339bc3c7f959d, store: [table=TestAcidGuarantees family=A region=43137c94133f8cabe05776420e1d74bb] 2024-11-07T15:55:09,729 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b920a5596912458c809339bc3c7f959d is 175, key is test_row_0/A:col10/1730994907665/Put/seqid=0 2024-11-07T15:55:09,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742241_1417 (size=22561) 2024-11-07T15:55:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:10,138 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b920a5596912458c809339bc3c7f959d 2024-11-07T15:55:10,144 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/509ea6b1e4154814b4636af1dbc76a3a is 50, key is test_row_0/B:col10/1730994907665/Put/seqid=0 2024-11-07T15:55:10,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742242_1418 (size=9857) 2024-11-07T15:55:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:10,548 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/509ea6b1e4154814b4636af1dbc76a3a 2024-11-07T15:55:10,554 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/8b873d4c02274766b2862ca3ba97eda9 is 50, key is test_row_0/C:col10/1730994907665/Put/seqid=0 2024-11-07T15:55:10,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742243_1419 (size=9857) 2024-11-07T15:55:10,957 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/8b873d4c02274766b2862ca3ba97eda9 2024-11-07T15:55:10,961 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/A/b920a5596912458c809339bc3c7f959d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b920a5596912458c809339bc3c7f959d 2024-11-07T15:55:10,964 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b920a5596912458c809339bc3c7f959d, entries=100, sequenceid=312, filesize=22.0 K 2024-11-07T15:55:10,966 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/B/509ea6b1e4154814b4636af1dbc76a3a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/509ea6b1e4154814b4636af1dbc76a3a 2024-11-07T15:55:10,969 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/509ea6b1e4154814b4636af1dbc76a3a, entries=100, sequenceid=312, filesize=9.6 K 2024-11-07T15:55:10,969 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/.tmp/C/8b873d4c02274766b2862ca3ba97eda9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8b873d4c02274766b2862ca3ba97eda9 2024-11-07T15:55:10,972 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8b873d4c02274766b2862ca3ba97eda9, entries=100, sequenceid=312, filesize=9.6 K 2024-11-07T15:55:10,973 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 43137c94133f8cabe05776420e1d74bb in 1657ms, sequenceid=312, compaction requested=true 2024-11-07T15:55:10,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1] to archive 2024-11-07T15:55:10,974 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:10,976 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/413fc05eb97c4517b9ff4f2c9a3aff2e 2024-11-07T15:55:10,977 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/cf485b90f08a42a5aa35e0d9797b4c14 2024-11-07T15:55:10,978 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/0a209971da9943729aee9727a090a938 2024-11-07T15:55:10,979 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a092080df9c6499e914cd4fc0efc857d 2024-11-07T15:55:10,980 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/42088dc653ec4fb5a4d3b7f7dd200d1d 2024-11-07T15:55:10,981 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/e8322df2e6a24c4a9a9497be18293904 2024-11-07T15:55:10,981 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/50c67aa25fa447b28d0d7ce57558ebe0 2024-11-07T15:55:10,982 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/a06bdf1ca3964b41990a4ff18d9ac316 2024-11-07T15:55:10,983 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/38e031e723504498b65b46f21988f1be 2024-11-07T15:55:10,984 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/df12cb867c1348c4b654c6e579f19476 2024-11-07T15:55:10,985 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/011b4f62cb494f979c91a0a99e58c6ca 2024-11-07T15:55:10,986 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/7ac2b3d86d1143fc969123b28b3ca35b 2024-11-07T15:55:10,986 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/9cddf2758df148e2b33f0d0f12899f31 2024-11-07T15:55:10,987 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6bf34f2cb92c46189b005630f907a25b 2024-11-07T15:55:10,988 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/f276c1f03eb442a8afd7302bd89a6740 2024-11-07T15:55:10,989 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b96fc74bd9ac4a63865a6c127959142f 2024-11-07T15:55:10,990 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/05049a21d27d45be82c485ce93618733 2024-11-07T15:55:10,991 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/94498544d1ad4c41a77fa86c247f40fc 2024-11-07T15:55:10,991 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/6c45295ce20a4f1da178bb8b9c4c92e1 2024-11-07T15:55:10,993 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/a36eafc12a124cf8b13452e8dd14d88c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/845c822649004b60b59dede25b27434e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8175e48de367461596b877da032c9a44, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b5063f46fc29479d97a57b34433c1e3e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b7b8c6d0138c4672a08891c3cb44b55c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f] to archive 2024-11-07T15:55:10,993 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:10,994 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/9ae1937152e949bba1bf8bc2e7afbb0e 2024-11-07T15:55:10,995 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/25ec50bc15dc4cf589c9f90712252574 2024-11-07T15:55:10,996 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/a36eafc12a124cf8b13452e8dd14d88c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/a36eafc12a124cf8b13452e8dd14d88c 2024-11-07T15:55:10,997 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fc805e2434ac4c118746368d264971dd 2024-11-07T15:55:10,998 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/fccecb7721c84a03ad14e73d487d9e55 2024-11-07T15:55:10,999 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/845c822649004b60b59dede25b27434e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/845c822649004b60b59dede25b27434e 2024-11-07T15:55:10,999 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/06a324184bb44f22b20c7763c52897f5 2024-11-07T15:55:11,000 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5e958470e94e31915f0ef83c52124e 2024-11-07T15:55:11,001 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8175e48de367461596b877da032c9a44 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8175e48de367461596b877da032c9a44 2024-11-07T15:55:11,002 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/162f07061e6c47fd96badbdd9539ed3c 2024-11-07T15:55:11,003 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/ec5c2eaf7ed4410195540bdbb59a1e65 2024-11-07T15:55:11,004 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/57a5af3432b548e6931a7396fa91eea1 2024-11-07T15:55:11,005 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b5063f46fc29479d97a57b34433c1e3e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b5063f46fc29479d97a57b34433c1e3e 2024-11-07T15:55:11,006 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/32699106ca65400d9f62c9f6282cbce1 2024-11-07T15:55:11,006 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/462676543be2438ba1b9e8840c3d49ba 2024-11-07T15:55:11,007 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b7b8c6d0138c4672a08891c3cb44b55c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/b7b8c6d0138c4672a08891c3cb44b55c 2024-11-07T15:55:11,008 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/66e1edabcfc847f5bd2caa6461d9613a 2024-11-07T15:55:11,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/8c8436055584408093d329a566f13a64 2024-11-07T15:55:11,009 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/c5d3b0363bd6438e924cb1b5451fdb0f 2024-11-07T15:55:11,011 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f736c9d3e72544b9b80bd83a49f43bad, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0690c0d5a8dd4865af77783a2ab182be, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e0213032970449eabae4fd079c5d9005, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/424e35afcffe415ba0b74d932b36cd8a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8777981f017d43808531985cfa97c189, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b] to archive 2024-11-07T15:55:11,011 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:11,012 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/042fdac84e0a4d1e9c45313b0a731e23 2024-11-07T15:55:11,013 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/fed1ee84284548a1baef5dfb5f720f1f 2024-11-07T15:55:11,014 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f736c9d3e72544b9b80bd83a49f43bad to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f736c9d3e72544b9b80bd83a49f43bad 2024-11-07T15:55:11,015 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0b2ceb185d8042098e01aecc2a761d1d 2024-11-07T15:55:11,016 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/86dff72e38644054b32934074e7987a4 2024-11-07T15:55:11,017 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0690c0d5a8dd4865af77783a2ab182be to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/0690c0d5a8dd4865af77783a2ab182be 2024-11-07T15:55:11,018 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e616beadc8414c5dae603d8ac5a92ba1 2024-11-07T15:55:11,019 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2f76955b7b114f3eae4f194308faa869 2024-11-07T15:55:11,019 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e0213032970449eabae4fd079c5d9005 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e0213032970449eabae4fd079c5d9005 2024-11-07T15:55:11,020 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/cacd327725164dc6acfabf939670abc6 2024-11-07T15:55:11,021 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afcdc1918b0d4f93be8bcb97e72169b8 2024-11-07T15:55:11,022 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/e598bb36abe6421ca71cad029a48b4b4 2024-11-07T15:55:11,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/424e35afcffe415ba0b74d932b36cd8a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/424e35afcffe415ba0b74d932b36cd8a 2024-11-07T15:55:11,023 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/c032d94cc3ae4a37aec23fa4e838b44d 2024-11-07T15:55:11,024 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/55a3a255a22643feb38ed64dc9edf4f3 2024-11-07T15:55:11,025 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8777981f017d43808531985cfa97c189 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8777981f017d43808531985cfa97c189 2024-11-07T15:55:11,026 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/04ef060d9fba4798a4a40329960c5e08 2024-11-07T15:55:11,027 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/afd44038f201442e87e863b8e2a968d8 2024-11-07T15:55:11,028 DEBUG [StoreCloser-TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/ac248c36bf174c2e95bc51dc975c495b 2024-11-07T15:55:11,031 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits/315.seqid, newMaxSeqId=315, maxSeqId=4 2024-11-07T15:55:11,032 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. 2024-11-07T15:55:11,032 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 43137c94133f8cabe05776420e1d74bb: Waiting for close lock at 1730994909295Running coprocessor pre-close hooks at 1730994909295Disabling compacts and flushes for region at 1730994909295Disabling writes for close at 1730994909316 (+21 ms)Obtaining lock to block concurrent updates at 1730994909316Preparing flush snapshotting stores in 43137c94133f8cabe05776420e1d74bb at 1730994909316Finished memstore snapshotting TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1730994909317 (+1 ms)Flushing stores of TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb. at 1730994909317Flushing 43137c94133f8cabe05776420e1d74bb/A: creating writer at 1730994909317Flushing mob file 43137c94133f8cabe05776420e1d74bb/A: appending metadata at 1730994909321 (+4 ms)Flushing mob file 43137c94133f8cabe05776420e1d74bb/A: closing flushed file at 1730994909321Flushing 43137c94133f8cabe05776420e1d74bb/A: appending metadata at 1730994909729 (+408 ms)Flushing 43137c94133f8cabe05776420e1d74bb/A: closing flushed file at 1730994909729Flushing 43137c94133f8cabe05776420e1d74bb/B: creating writer at 1730994910142 (+413 ms)Flushing 43137c94133f8cabe05776420e1d74bb/B: appending metadata at 1730994910144 (+2 ms)Flushing 43137c94133f8cabe05776420e1d74bb/B: closing flushed file at 1730994910144Flushing 43137c94133f8cabe05776420e1d74bb/C: creating writer at 1730994910551 (+407 ms)Flushing 43137c94133f8cabe05776420e1d74bb/C: appending metadata at 1730994910553 (+2 ms)Flushing 43137c94133f8cabe05776420e1d74bb/C: closing flushed file at 1730994910553Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@679c9706: reopening flushed file at 1730994910961 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d9aaa1: reopening flushed file at 1730994910964 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bbbdd75: reopening flushed file at 1730994910969 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 43137c94133f8cabe05776420e1d74bb in 1657ms, sequenceid=312, compaction requested=true at 1730994910973 (+4 ms)Writing region close event to WAL at 1730994911029 (+56 ms)Running coprocessor post-close hooks at 1730994911032 (+3 ms)Closed at 1730994911032 2024-11-07T15:55:11,034 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,034 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=43137c94133f8cabe05776420e1d74bb, regionState=CLOSED 2024-11-07T15:55:11,036 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:11,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-07T15:55:11,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 43137c94133f8cabe05776420e1d74bb, server=c2682f91c1e2,42309,1730994796599 in 1.8930 sec 2024-11-07T15:55:11,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-07T15:55:11,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=43137c94133f8cabe05776420e1d74bb, UNASSIGN in 1.8980 sec 2024-11-07T15:55:11,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-07T15:55:11,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-07T15:55:11,043 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994911043"}]},"ts":"1730994911043"} 2024-11-07T15:55:11,044 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:55:11,044 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:55:11,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9100 sec 2024-11-07T15:55:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-07T15:55:11,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:11,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:55:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,285 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T15:55:11,286 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,289 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,290 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits] 2024-11-07T15:55:11,292 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/16b9ad70ed674eb4be47fa916023a5a9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/16b9ad70ed674eb4be47fa916023a5a9 2024-11-07T15:55:11,293 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/3f91bfc7b8014a22b179baa3f3627a9d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/3f91bfc7b8014a22b179baa3f3627a9d 2024-11-07T15:55:11,294 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/53445ab511864c4f813e039033c9a2e6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/53445ab511864c4f813e039033c9a2e6 2024-11-07T15:55:11,295 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b920a5596912458c809339bc3c7f959d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/A/b920a5596912458c809339bc3c7f959d 2024-11-07T15:55:11,297 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/09c5516440d148b7807ea431354ab775 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/09c5516440d148b7807ea431354ab775 2024-11-07T15:55:11,298 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/2a138319d5e945839a399d3c3c52fd1f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/2a138319d5e945839a399d3c3c52fd1f 2024-11-07T15:55:11,299 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/509ea6b1e4154814b4636af1dbc76a3a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/509ea6b1e4154814b4636af1dbc76a3a 2024-11-07T15:55:11,300 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/7d1b2ede5cd046cbb77188deef4c2628 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/B/7d1b2ede5cd046cbb77188deef4c2628 2024-11-07T15:55:11,302 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2c83cf9c3a9b44f7ab09e3a976cedab9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/2c83cf9c3a9b44f7ab09e3a976cedab9 2024-11-07T15:55:11,303 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8b873d4c02274766b2862ca3ba97eda9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/8b873d4c02274766b2862ca3ba97eda9 2024-11-07T15:55:11,304 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/a69228b91f5b45c28d0632141ef6a441 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/a69228b91f5b45c28d0632141ef6a441 2024-11-07T15:55:11,305 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f77a8d27ea5e4f88a5f024bc5d75db14 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/C/f77a8d27ea5e4f88a5f024bc5d75db14 2024-11-07T15:55:11,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits/315.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb/recovered.edits/315.seqid 2024-11-07T15:55:11,307 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,307 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:55:11,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:55:11,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:55:11,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ec70c9a1d42433ab4ac42936678ccbc_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411070ec70c9a1d42433ab4ac42936678ccbc_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718654680561e4faeb2e552fa3f51339b_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110718654680561e4faeb2e552fa3f51339b_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,312 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071a963631517049a19be3723759848a50_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411071a963631517049a19be3723759848a50_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,313 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072b8c199332b14739a097308ba8f80ada_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072b8c199332b14739a097308ba8f80ada_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,314 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ca736b0ccbb476bb51765f4111e4665_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072ca736b0ccbb476bb51765f4111e4665_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072d867173520d484d8e6cd0abe57fb074_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411072d867173520d484d8e6cd0abe57fb074_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,316 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736883de2fa7e42f69c1eeebf38e15951_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110736883de2fa7e42f69c1eeebf38e15951_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749113f4c790b43c497675c1b6771666e_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110749113f4c790b43c497675c1b6771666e_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,318 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076cdc79f8792a4abdb41f5ab3eb678302_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076cdc79f8792a4abdb41f5ab3eb678302_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,319 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078644be73dc6a41a88c11a4f768762634_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078644be73dc6a41a88c11a4f768762634_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,320 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ceac58fce2e466abe679d7818ce531c_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411078ceac58fce2e466abe679d7818ce531c_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,321 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079c6f162d328947568a58c53f05a55324_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079c6f162d328947568a58c53f05a55324_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,322 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a0092f38342043dc87244aba04fe814f_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107a0092f38342043dc87244aba04fe814f_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,323 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c92a761c4ba141baa23ece3fae43bdef_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107c92a761c4ba141baa23ece3fae43bdef_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,324 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3137bc5d34f43a5883354706eb910d3_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d3137bc5d34f43a5883354706eb910d3_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,325 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7db2238c80a470b892317b7d5e0e5e2_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107d7db2238c80a470b892317b7d5e0e5e2_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6a30977b3664363b7e50241dab79de5_43137c94133f8cabe05776420e1d74bb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107e6a30977b3664363b7e50241dab79de5_43137c94133f8cabe05776420e1d74bb 2024-11-07T15:55:11,326 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:55:11,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,330 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:55:11,332 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:55:11,333 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,333 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:55:11,333 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994911333"}]},"ts":"9223372036854775807"} 2024-11-07T15:55:11,335 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:55:11,335 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 43137c94133f8cabe05776420e1d74bb, NAME => 'TestAcidGuarantees,,1730994883926.43137c94133f8cabe05776420e1d74bb.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:55:11,335 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:55:11,335 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994911335"}]},"ts":"9223372036854775807"} 2024-11-07T15:55:11,336 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:55:11,337 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,338 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-07T15:55:11,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-07T15:55:11,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:55:11,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:11,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 239) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_816498657_22 at /127.0.0.1:44080 [Waiting for operation #450] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-9504082_22 at /127.0.0.1:57286 [Waiting for operation #569] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_816498657_22 at /127.0.0.1:44082 [Waiting for operation #447] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-9504082_22 at /127.0.0.1:48614 [Waiting for operation #1727] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=320 (was 279) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5511 (was 5570) 2024-11-07T15:55:11,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=320, ProcessCount=11, AvailableMemoryMB=5511 2024-11-07T15:55:11,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:55:11,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:55:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:11,414 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:55:11,414 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:11,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-07T15:55:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:11,415 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:55:11,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742244_1420 (size=959) 2024-11-07T15:55:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:11,821 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:55:11,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742245_1421 (size=53) 2024-11-07T15:55:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8c07d911036021999ccdca9d657aee9b, disabling compactions & flushes 2024-11-07T15:55:12,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. after waiting 0 ms 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,226 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,226 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8c07d911036021999ccdca9d657aee9b: Waiting for close lock at 1730994912226Disabling compacts and flushes for region at 1730994912226Disabling writes for close at 1730994912226Writing region close event to WAL at 1730994912226Closed at 1730994912226 2024-11-07T15:55:12,227 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:55:12,228 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994912228"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994912228"}]},"ts":"1730994912228"} 2024-11-07T15:55:12,229 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:55:12,230 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:55:12,230 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994912230"}]},"ts":"1730994912230"} 2024-11-07T15:55:12,231 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:55:12,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, ASSIGN}] 2024-11-07T15:55:12,232 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, ASSIGN 2024-11-07T15:55:12,233 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:55:12,383 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=8c07d911036021999ccdca9d657aee9b, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:12,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, ASSIGN because future has completed 2024-11-07T15:55:12,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:12,539 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,539 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:55:12,540 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,541 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:55:12,541 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,541 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,542 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,543 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c07d911036021999ccdca9d657aee9b columnFamilyName A 2024-11-07T15:55:12,543 DEBUG [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:12,543 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(327): Store=8c07d911036021999ccdca9d657aee9b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:12,543 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,544 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c07d911036021999ccdca9d657aee9b columnFamilyName B 2024-11-07T15:55:12,544 DEBUG [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:12,545 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(327): Store=8c07d911036021999ccdca9d657aee9b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:12,545 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,545 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c07d911036021999ccdca9d657aee9b columnFamilyName C 2024-11-07T15:55:12,545 DEBUG [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:12,546 INFO [StoreOpener-8c07d911036021999ccdca9d657aee9b-1 {}] regionserver.HStore(327): Store=8c07d911036021999ccdca9d657aee9b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:12,546 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,546 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,547 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,547 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,547 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,548 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:55:12,549 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,550 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:55:12,550 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 8c07d911036021999ccdca9d657aee9b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62643294, jitterRate=-0.06654217839241028}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:55:12,550 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:12,551 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 8c07d911036021999ccdca9d657aee9b: Running coprocessor pre-open hook at 1730994912541Writing region info on filesystem at 1730994912541Initializing all the Stores at 1730994912541Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994912541Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994912541Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994912541Cleaning up temporary data from old regions at 1730994912547 (+6 ms)Running coprocessor post-open hooks at 1730994912550 (+3 ms)Region opened successfully at 1730994912551 (+1 ms) 2024-11-07T15:55:12,551 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., pid=120, masterSystemTime=1730994912537 2024-11-07T15:55:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:12,553 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,553 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:12,553 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=8c07d911036021999ccdca9d657aee9b, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:12,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:12,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-07T15:55:12,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 in 170 msec 2024-11-07T15:55:12,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-07T15:55:12,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, ASSIGN in 326 msec 2024-11-07T15:55:12,559 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:55:12,559 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994912559"}]},"ts":"1730994912559"} 2024-11-07T15:55:12,560 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:55:12,560 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:55:12,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-07T15:55:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-07T15:55:13,564 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:13,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ded0f15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,565 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,565 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,566 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3718d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,567 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4353fc86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,568 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,570 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:55:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:55:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@120121b4 2024-11-07T15:55:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:55:13,572 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:55:13,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d6d1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,574 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b7c261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,575 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5193c41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,576 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@83ca52f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,578 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b76e9b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205b33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,579 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb783c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54718, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4262410e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e100fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,582 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dda475f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,583 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754df36f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e40afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,584 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cb4e29b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,585 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7026b5ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,586 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259a56de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,586 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,587 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7668c1b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,588 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4940e86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4cbf6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d182797, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,590 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66bf1627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c83334c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d4b806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78cf4766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3532b5c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305e9451, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e8849b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3088fab0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:13,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:13,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c79379d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:13,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:13,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73786d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:13,598 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,599 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,599 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,599 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,599 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,600 DEBUG [Thread-1901 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,601 DEBUG [Thread-1912 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,601 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,601 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,602 DEBUG [Thread-1909 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,603 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,603 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:13,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33112, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33150, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-07T15:55:13,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,608 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2] 2024-11-07T15:55:13,610 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:13,611 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:13,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:13,616 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:13,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:55:13,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/2969c4e5011e42d88de7f8557c1637fb is 50, key is test_row_0/A:col10/1730994913608/Put/seqid=0 2024-11-07T15:55:13,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742246_1422 (size=16681) 2024-11-07T15:55:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994973655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994973656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994973659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994973668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994973669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:13,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:13,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994973773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994973773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994973773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,781 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994973793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994973793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:13,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:13,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994973983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:13,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994973983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:13,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994973993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994974003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994974004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/2969c4e5011e42d88de7f8557c1637fb 2024-11-07T15:55:14,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7ec63817e06443dc84e1c757ff9aed76 is 50, key is test_row_0/B:col10/1730994913608/Put/seqid=0 2024-11-07T15:55:14,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742247_1423 (size=12001) 2024-11-07T15:55:14,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994974293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,298 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994974302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994974313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994974323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994974323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7ec63817e06443dc84e1c757ff9aed76 2024-11-07T15:55:14,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/7c69a5d497a24631a1b012149c40bf75 is 50, key is test_row_0/C:col10/1730994913608/Put/seqid=0 2024-11-07T15:55:14,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742248_1424 (size=12001) 2024-11-07T15:55:14,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:14,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994974803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994974813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994974823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:14,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994974833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994974833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:14,926 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:55:14,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/7c69a5d497a24631a1b012149c40bf75 2024-11-07T15:55:14,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/2969c4e5011e42d88de7f8557c1637fb as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb 2024-11-07T15:55:14,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb, entries=250, sequenceid=14, filesize=16.3 K 2024-11-07T15:55:14,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7ec63817e06443dc84e1c757ff9aed76 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76 2024-11-07T15:55:14,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:55:14,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/7c69a5d497a24631a1b012149c40bf75 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75 2024-11-07T15:55:14,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75, entries=150, sequenceid=14, filesize=11.7 K 2024-11-07T15:55:14,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8c07d911036021999ccdca9d657aee9b in 1337ms, sequenceid=14, compaction requested=false 2024-11-07T15:55:14,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:14,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-07T15:55:14,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:14,988 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:55:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/09d6817c37614be5bfe517798ff1c6fe is 50, key is test_row_0/A:col10/1730994913656/Put/seqid=0 2024-11-07T15:55:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742249_1425 (size=12001) 2024-11-07T15:55:14,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/09d6817c37614be5bfe517798ff1c6fe 2024-11-07T15:55:15,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/b74575b528544f89a70b298eb1108587 is 50, key is test_row_0/B:col10/1730994913656/Put/seqid=0 2024-11-07T15:55:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742250_1426 (size=12001) 2024-11-07T15:55:15,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/b74575b528544f89a70b298eb1108587 2024-11-07T15:55:15,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/c2f8ddefda464dc596ab15470f3cef01 is 50, key is test_row_0/C:col10/1730994913656/Put/seqid=0 2024-11-07T15:55:15,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742251_1427 (size=12001) 2024-11-07T15:55:15,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/c2f8ddefda464dc596ab15470f3cef01 2024-11-07T15:55:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/09d6817c37614be5bfe517798ff1c6fe as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe 2024-11-07T15:55:15,427 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T15:55:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/b74575b528544f89a70b298eb1108587 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587 2024-11-07T15:55:15,430 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T15:55:15,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/c2f8ddefda464dc596ab15470f3cef01 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01 2024-11-07T15:55:15,434 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01, entries=150, sequenceid=37, filesize=11.7 K 2024-11-07T15:55:15,435 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 8c07d911036021999ccdca9d657aee9b in 447ms, sequenceid=37, compaction requested=false 2024-11-07T15:55:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-07T15:55:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-07T15:55:15,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-07T15:55:15,439 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-07T15:55:15,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8370 sec 2024-11-07T15:55:15,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-07T15:55:15,753 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:15,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-07T15:55:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:15,756 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:15,756 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:15,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:15,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:15,845 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:15,845 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:55:15,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/385cedc29a464f068d2de69e04c3370e is 50, key is test_row_0/A:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:15,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742252_1428 (size=16681) 2024-11-07T15:55:15,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:15,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:15,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:15,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:15,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:15,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994975918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:15,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994975919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:15,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994975920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994975932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:15,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:15,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994975934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994976043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994976043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994976043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994976053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994976063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:16,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994976263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994976263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/385cedc29a464f068d2de69e04c3370e 2024-11-07T15:55:16,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994976263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/192e21f08c0249a8b707fe5a59c4a436 is 50, key is test_row_0/B:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:16,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742253_1429 (size=12001) 2024-11-07T15:55:16,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994976273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994976283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:16,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,554 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:55:16,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994976573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994976573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994976573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994976584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994976603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:16,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/192e21f08c0249a8b707fe5a59c4a436 2024-11-07T15:55:16,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/f7a11e764ac64bd485741d2cbf69c192 is 50, key is test_row_0/C:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:16,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742254_1430 (size=12001) 2024-11-07T15:55:16,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:16,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:16,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:16,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:16,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/f7a11e764ac64bd485741d2cbf69c192 2024-11-07T15:55:17,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/385cedc29a464f068d2de69e04c3370e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e 2024-11-07T15:55:17,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e, entries=250, sequenceid=48, filesize=16.3 K 2024-11-07T15:55:17,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/192e21f08c0249a8b707fe5a59c4a436 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436 2024-11-07T15:55:17,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994977093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436, entries=150, sequenceid=48, filesize=11.7 K 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994977093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994977093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/f7a11e764ac64bd485741d2cbf69c192 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192 2024-11-07T15:55:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,102 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192, entries=150, sequenceid=48, filesize=11.7 K 2024-11-07T15:55:17,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8c07d911036021999ccdca9d657aee9b in 1260ms, sequenceid=48, compaction requested=true 2024-11-07T15:55:17,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:17,106 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:17,106 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:17,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:17,106 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:17,106 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:17,107 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,107 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,107 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=44.3 K 2024-11-07T15:55:17,107 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.2 K 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2969c4e5011e42d88de7f8557c1637fb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994913607 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec63817e06443dc84e1c757ff9aed76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994913608 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b74575b528544f89a70b298eb1108587, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730994913656 2024-11-07T15:55:17,107 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09d6817c37614be5bfe517798ff1c6fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730994913656 2024-11-07T15:55:17,108 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 192e21f08c0249a8b707fe5a59c4a436, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915834 2024-11-07T15:55:17,108 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 385cedc29a464f068d2de69e04c3370e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915833 2024-11-07T15:55:17,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:17,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:17,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-07T15:55:17,116 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:17,116 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/841e1cef500343eab551d7cfd4802d7b is 50, key is test_row_0/B:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:17,117 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:17,117 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/5015e6bfc8c040edb554a342aa6df681 is 50, key is test_row_0/A:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:17,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a822df6a58214dde896b10b2ca81c2fe is 50, key is test_row_0/A:col10/1730994915897/Put/seqid=0 2024-11-07T15:55:17,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742256_1432 (size=12104) 2024-11-07T15:55:17,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742255_1431 (size=12104) 2024-11-07T15:55:17,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742257_1433 (size=14341) 2024-11-07T15:55:17,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:17,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a822df6a58214dde896b10b2ca81c2fe 2024-11-07T15:55:17,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/cb1ccef920424d66abe5552906a9c023 is 50, key is test_row_0/B:col10/1730994915897/Put/seqid=0 2024-11-07T15:55:17,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742258_1434 (size=12001) 2024-11-07T15:55:17,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/cb1ccef920424d66abe5552906a9c023 2024-11-07T15:55:17,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/b2e548f589fc46c4bf8f5f150c295088 is 50, key is test_row_0/C:col10/1730994915897/Put/seqid=0 2024-11-07T15:55:17,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742259_1435 (size=12001) 2024-11-07T15:55:17,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994977147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994977155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994977263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994977273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:17,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994977483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994977493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,533 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/5015e6bfc8c040edb554a342aa6df681 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/5015e6bfc8c040edb554a342aa6df681 2024-11-07T15:55:17,536 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/841e1cef500343eab551d7cfd4802d7b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/841e1cef500343eab551d7cfd4802d7b 2024-11-07T15:55:17,537 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into 5015e6bfc8c040edb554a342aa6df681(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:17,537 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:17,537 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=13, startTime=1730994917105; duration=0sec 2024-11-07T15:55:17,537 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:17,538 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:17,538 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:17,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:17,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:17,539 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,539 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.2 K 2024-11-07T15:55:17,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c69a5d497a24631a1b012149c40bf75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1730994913608 2024-11-07T15:55:17,540 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2f8ddefda464dc596ab15470f3cef01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1730994913656 2024-11-07T15:55:17,540 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7a11e764ac64bd485741d2cbf69c192, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915834 2024-11-07T15:55:17,540 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 841e1cef500343eab551d7cfd4802d7b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:17,541 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:17,541 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=13, startTime=1730994917106; duration=0sec 2024-11-07T15:55:17,541 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:17,541 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:17,546 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:17,546 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/826aba1b39ac47bfad509a05793ecfa7 is 50, key is test_row_0/C:col10/1730994915834/Put/seqid=0 2024-11-07T15:55:17,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742260_1436 (size=12104) 2024-11-07T15:55:17,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/b2e548f589fc46c4bf8f5f150c295088 2024-11-07T15:55:17,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a822df6a58214dde896b10b2ca81c2fe as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe 2024-11-07T15:55:17,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe, entries=200, sequenceid=74, filesize=14.0 K 2024-11-07T15:55:17,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/cb1ccef920424d66abe5552906a9c023 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023 2024-11-07T15:55:17,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:55:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/b2e548f589fc46c4bf8f5f150c295088 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088 2024-11-07T15:55:17,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088, entries=150, sequenceid=74, filesize=11.7 K 2024-11-07T15:55:17,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8c07d911036021999ccdca9d657aee9b in 458ms, sequenceid=74, compaction requested=false 2024-11-07T15:55:17,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:17,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-07T15:55:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:17,593 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-07T15:55:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/74af0a9879cc4fdb8171549bba88a601 is 50, key is test_row_0/A:col10/1730994917146/Put/seqid=0 2024-11-07T15:55:17,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742261_1437 (size=9657) 2024-11-07T15:55:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:17,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:17,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994977899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:17,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:17,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994977902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:17,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:17,954 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/826aba1b39ac47bfad509a05793ecfa7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/826aba1b39ac47bfad509a05793ecfa7 2024-11-07T15:55:17,960 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into 826aba1b39ac47bfad509a05793ecfa7(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:17,960 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:17,960 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=13, startTime=1730994917106; duration=0sec 2024-11-07T15:55:17,960 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:17,960 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:18,002 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/74af0a9879cc4fdb8171549bba88a601 2024-11-07T15:55:18,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/0cf4e1d4c28d43c6825fc3f86ef67ede is 50, key is test_row_0/B:col10/1730994917146/Put/seqid=0 2024-11-07T15:55:18,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742262_1438 (size=9657) 2024-11-07T15:55:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994978013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994978023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994978113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994978113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994978114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,118 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,118 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,119 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,199 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e] to archive 2024-11-07T15:55:18,200 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:18,201 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/2969c4e5011e42d88de7f8557c1637fb 2024-11-07T15:55:18,202 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/09d6817c37614be5bfe517798ff1c6fe 2024-11-07T15:55:18,204 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/385cedc29a464f068d2de69e04c3370e 2024-11-07T15:55:18,204 WARN [regionserver/c2682f91c1e2:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-07T15:55:18,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436] to archive 2024-11-07T15:55:18,205 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:18,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7ec63817e06443dc84e1c757ff9aed76 2024-11-07T15:55:18,207 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/b74575b528544f89a70b298eb1108587 2024-11-07T15:55:18,209 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/192e21f08c0249a8b707fe5a59c4a436 2024-11-07T15:55:18,210 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192] to archive 2024-11-07T15:55:18,211 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:18,212 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/7c69a5d497a24631a1b012149c40bf75 2024-11-07T15:55:18,213 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/c2f8ddefda464dc596ab15470f3cef01 2024-11-07T15:55:18,214 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c2682f91c1e2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/f7a11e764ac64bd485741d2cbf69c192 2024-11-07T15:55:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994978233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994978233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,413 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/0cf4e1d4c28d43c6825fc3f86ef67ede 2024-11-07T15:55:18,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/16f68d073e9f44209799f07a9bb492d4 is 50, key is test_row_0/C:col10/1730994917146/Put/seqid=0 2024-11-07T15:55:18,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742263_1439 (size=9657) 2024-11-07T15:55:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994978553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994978553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:18,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:18,823 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/16f68d073e9f44209799f07a9bb492d4 2024-11-07T15:55:18,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/74af0a9879cc4fdb8171549bba88a601 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601 2024-11-07T15:55:18,830 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601, entries=100, sequenceid=86, filesize=9.4 K 2024-11-07T15:55:18,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/0cf4e1d4c28d43c6825fc3f86ef67ede as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede 2024-11-07T15:55:18,834 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede, entries=100, sequenceid=86, filesize=9.4 K 2024-11-07T15:55:18,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/16f68d073e9f44209799f07a9bb492d4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4 2024-11-07T15:55:18,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4, entries=100, sequenceid=86, filesize=9.4 K 2024-11-07T15:55:18,839 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 8c07d911036021999ccdca9d657aee9b in 1246ms, sequenceid=86, compaction requested=true 2024-11-07T15:55:18,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:18,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-07T15:55:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-07T15:55:18,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-07T15:55:18,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0850 sec 2024-11-07T15:55:18,844 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.0890 sec 2024-11-07T15:55:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:19,078 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:19,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-07T15:55:19,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/aa295fa471f340d4b005d904940d30bc is 50, key is test_row_0/A:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:19,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742264_1440 (size=14341) 2024-11-07T15:55:19,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/aa295fa471f340d4b005d904940d30bc 2024-11-07T15:55:19,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/3328d15410ab49ec9c42c0420e31b99f is 50, key is test_row_0/B:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:19,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994979099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994979112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,116 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742265_1441 (size=12001) 2024-11-07T15:55:19,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994979223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994979223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994979443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994979443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,452 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/3328d15410ab49ec9c42c0420e31b99f 2024-11-07T15:55:19,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/10794648b8954b75b1cd98469cdded90 is 50, key is test_row_0/C:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:19,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742266_1442 (size=12001) 2024-11-07T15:55:19,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994979763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994979763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-07T15:55:19,933 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:19,934 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-07T15:55:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:19,936 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:19,937 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:19,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:19,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/10794648b8954b75b1cd98469cdded90 2024-11-07T15:55:19,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/aa295fa471f340d4b005d904940d30bc as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc 2024-11-07T15:55:19,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc, entries=200, sequenceid=114, filesize=14.0 K 2024-11-07T15:55:19,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/3328d15410ab49ec9c42c0420e31b99f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f 2024-11-07T15:55:19,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T15:55:19,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/10794648b8954b75b1cd98469cdded90 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90 2024-11-07T15:55:19,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90, entries=150, sequenceid=114, filesize=11.7 K 2024-11-07T15:55:19,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8c07d911036021999ccdca9d657aee9b in 876ms, sequenceid=114, compaction requested=true 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:19,955 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:19,955 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:19,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:19,956 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50443 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:19,956 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:19,956 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:19,956 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:19,956 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:19,956 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:19,956 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/841e1cef500343eab551d7cfd4802d7b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=44.7 K 2024-11-07T15:55:19,956 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/5015e6bfc8c040edb554a342aa6df681, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=49.3 K 2024-11-07T15:55:19,956 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 841e1cef500343eab551d7cfd4802d7b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915834 2024-11-07T15:55:19,957 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5015e6bfc8c040edb554a342aa6df681, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915834 2024-11-07T15:55:19,957 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a822df6a58214dde896b10b2ca81c2fe, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730994915897 2024-11-07T15:55:19,957 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting cb1ccef920424d66abe5552906a9c023, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730994915897 2024-11-07T15:55:19,957 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74af0a9879cc4fdb8171549bba88a601, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730994917146 2024-11-07T15:55:19,957 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf4e1d4c28d43c6825fc3f86ef67ede, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730994917146 2024-11-07T15:55:19,958 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa295fa471f340d4b005d904940d30bc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:19,958 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3328d15410ab49ec9c42c0420e31b99f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:19,967 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:19,967 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:19,967 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/184bdcf36a934513b6e1c762e56033fc is 50, key is test_row_0/B:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:19,967 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/fc09d5f126dd491883a3593f1697c172 is 50, key is test_row_0/A:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:19,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742267_1443 (size=12139) 2024-11-07T15:55:19,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742268_1444 (size=12139) 2024-11-07T15:55:19,982 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/fc09d5f126dd491883a3593f1697c172 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/fc09d5f126dd491883a3593f1697c172 2024-11-07T15:55:19,986 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into fc09d5f126dd491883a3593f1697c172(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:19,986 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:19,987 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=12, startTime=1730994919955; duration=0sec 2024-11-07T15:55:19,987 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:19,987 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:19,987 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:19,988 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45763 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:19,988 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:19,988 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:19,988 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/826aba1b39ac47bfad509a05793ecfa7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=44.7 K 2024-11-07T15:55:19,988 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 826aba1b39ac47bfad509a05793ecfa7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1730994915834 2024-11-07T15:55:19,988 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e548f589fc46c4bf8f5f150c295088, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1730994915897 2024-11-07T15:55:19,989 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16f68d073e9f44209799f07a9bb492d4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1730994917146 2024-11-07T15:55:19,990 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10794648b8954b75b1cd98469cdded90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:19,999 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:19,999 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/bcbbede10763432ea7bc094bc8bd880d is 50, key is test_row_0/C:col10/1730994917886/Put/seqid=0 2024-11-07T15:55:20,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742269_1445 (size=12139) 2024-11-07T15:55:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:20,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-07T15:55:20,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:20,090 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-07T15:55:20,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/e22b286b98ca4b9181f8755835771915 is 50, key is test_row_0/A:col10/1730994919088/Put/seqid=0 2024-11-07T15:55:20,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742270_1446 (size=12001) 2024-11-07T15:55:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:20,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:20,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994980206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,220 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994980212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994980214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994980283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994980283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994980323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994980333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994980333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,377 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/184bdcf36a934513b6e1c762e56033fc as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/184bdcf36a934513b6e1c762e56033fc 2024-11-07T15:55:20,381 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 184bdcf36a934513b6e1c762e56033fc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:20,381 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:20,381 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=12, startTime=1730994919955; duration=0sec 2024-11-07T15:55:20,381 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:20,381 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:20,407 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/bcbbede10763432ea7bc094bc8bd880d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/bcbbede10763432ea7bc094bc8bd880d 2024-11-07T15:55:20,411 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into bcbbede10763432ea7bc094bc8bd880d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:20,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:20,411 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=12, startTime=1730994919955; duration=0sec 2024-11-07T15:55:20,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:20,411 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:20,506 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/e22b286b98ca4b9181f8755835771915 2024-11-07T15:55:20,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/563432b1caf542febfc7817aa9224865 is 50, key is test_row_0/B:col10/1730994919088/Put/seqid=0 2024-11-07T15:55:20,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742271_1447 (size=12001) 2024-11-07T15:55:20,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994980543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994980543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994980543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994980863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994980863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994980863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:20,918 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/563432b1caf542febfc7817aa9224865 2024-11-07T15:55:20,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/ecd8d0a6ebe24a639b87d785a23a705e is 50, key is test_row_0/C:col10/1730994919088/Put/seqid=0 2024-11-07T15:55:20,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742272_1448 (size=12001) 2024-11-07T15:55:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:21,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994981303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,307 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994981303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,312 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:21,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,330 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/ecd8d0a6ebe24a639b87d785a23a705e 2024-11-07T15:55:21,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/e22b286b98ca4b9181f8755835771915 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915 2024-11-07T15:55:21,337 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:55:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/563432b1caf542febfc7817aa9224865 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865 2024-11-07T15:55:21,341 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:55:21,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/ecd8d0a6ebe24a639b87d785a23a705e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e 2024-11-07T15:55:21,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e, entries=150, sequenceid=124, filesize=11.7 K 2024-11-07T15:55:21,345 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 8c07d911036021999ccdca9d657aee9b in 1255ms, sequenceid=124, compaction requested=false 2024-11-07T15:55:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:21,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-07T15:55:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-07T15:55:21,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-07T15:55:21,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4100 sec 2024-11-07T15:55:21,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4150 sec 2024-11-07T15:55:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:21,390 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:21,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-07T15:55:21,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/706fce3e84644f2ca24439cdc52e0470 is 50, key is test_row_0/A:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:21,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742273_1449 (size=12151) 2024-11-07T15:55:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994981393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994981398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994981401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994981513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994981513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994981522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994981733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994981733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994981733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:21,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:21,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:21,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/706fce3e84644f2ca24439cdc52e0470 2024-11-07T15:55:21,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7741ece7693c49a79178c42e4f470068 is 50, key is test_row_0/B:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:21,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742274_1450 (size=12151) 2024-11-07T15:55:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994982043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994982043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994982053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-07T15:55:22,083 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:22,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-07T15:55:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:22,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:22,086 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:22,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:22,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7741ece7693c49a79178c42e4f470068 2024-11-07T15:55:22,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e9895e6f26e54de9b6e622ddf134f2db is 50, key is test_row_0/C:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:22,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742275_1451 (size=12151) 2024-11-07T15:55:22,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:55:22,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:55:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:22,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:55:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994982553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994982563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994982563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:22,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e9895e6f26e54de9b6e622ddf134f2db 2024-11-07T15:55:22,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/706fce3e84644f2ca24439cdc52e0470 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470 2024-11-07T15:55:22,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T15:55:22,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/7741ece7693c49a79178c42e4f470068 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068 2024-11-07T15:55:22,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T15:55:22,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e9895e6f26e54de9b6e622ddf134f2db as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db 2024-11-07T15:55:22,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db, entries=150, sequenceid=155, filesize=11.9 K 2024-11-07T15:55:22,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 8c07d911036021999ccdca9d657aee9b in 1247ms, sequenceid=155, compaction requested=true 2024-11-07T15:55:22,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:22,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:22,638 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:22,638 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,638 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,639 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/184bdcf36a934513b6e1c762e56033fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.4 K 2024-11-07T15:55:22,639 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/fc09d5f126dd491883a3593f1697c172, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.4 K 2024-11-07T15:55:22,639 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc09d5f126dd491883a3593f1697c172, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:22,639 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 184bdcf36a934513b6e1c762e56033fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:22,639 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 563432b1caf542febfc7817aa9224865, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730994919088 2024-11-07T15:55:22,640 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e22b286b98ca4b9181f8755835771915, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730994919088 2024-11-07T15:55:22,640 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 7741ece7693c49a79178c42e4f470068, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:22,640 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 706fce3e84644f2ca24439cdc52e0470, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:22,646 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:22,646 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:22,646 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/ab40536e6c7f4503ae993c7e462e50f7 is 50, key is test_row_0/B:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:22,647 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/1bc30153d8674a439df3ba0006eb585a is 50, key is test_row_0/A:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742276_1452 (size=12391) 2024-11-07T15:55:22,662 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/ab40536e6c7f4503ae993c7e462e50f7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ab40536e6c7f4503ae993c7e462e50f7 2024-11-07T15:55:22,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742277_1453 (size=12391) 2024-11-07T15:55:22,669 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into ab40536e6c7f4503ae993c7e462e50f7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:22,669 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:22,669 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=13, startTime=1730994922638; duration=0sec 2024-11-07T15:55:22,669 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:22,669 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:22,670 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:22,670 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:22,670 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:22,670 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,671 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/bcbbede10763432ea7bc094bc8bd880d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.4 K 2024-11-07T15:55:22,671 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting bcbbede10763432ea7bc094bc8bd880d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1730994917885 2024-11-07T15:55:22,671 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd8d0a6ebe24a639b87d785a23a705e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1730994919088 2024-11-07T15:55:22,671 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e9895e6f26e54de9b6e622ddf134f2db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:22,677 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:22,678 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/ff728eb574e94256a8e7b470cf2a1ee2 is 50, key is test_row_0/C:col10/1730994921388/Put/seqid=0 2024-11-07T15:55:22,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742278_1454 (size=12391) 2024-11-07T15:55:22,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-07T15:55:22,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:22,698 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-07T15:55:22,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/6c15c8ed034f456bbc1867cdc3964f68 is 50, key is test_row_1/A:col10/1730994921391/Put/seqid=0 2024-11-07T15:55:22,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742279_1455 (size=9757) 2024-11-07T15:55:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:23,072 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/1bc30153d8674a439df3ba0006eb585a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/1bc30153d8674a439df3ba0006eb585a 2024-11-07T15:55:23,076 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into 1bc30153d8674a439df3ba0006eb585a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:23,076 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:23,076 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=13, startTime=1730994922637; duration=0sec 2024-11-07T15:55:23,076 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:23,076 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:23,087 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/ff728eb574e94256a8e7b470cf2a1ee2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ff728eb574e94256a8e7b470cf2a1ee2 2024-11-07T15:55:23,091 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into ff728eb574e94256a8e7b470cf2a1ee2(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:23,091 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:23,091 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=13, startTime=1730994922638; duration=0sec 2024-11-07T15:55:23,092 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:23,092 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:23,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/6c15c8ed034f456bbc1867cdc3964f68 2024-11-07T15:55:23,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2723c98127a449258060d5583deb5012 is 50, key is test_row_1/B:col10/1730994921391/Put/seqid=0 2024-11-07T15:55:23,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742280_1456 (size=9757) 2024-11-07T15:55:23,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:23,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994983459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994983460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,524 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2723c98127a449258060d5583deb5012 2024-11-07T15:55:23,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/2bd5920451174084adc6de4c5957e64f is 50, key is test_row_1/C:col10/1730994921391/Put/seqid=0 2024-11-07T15:55:23,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742281_1457 (size=9757) 2024-11-07T15:55:23,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994983563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,571 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:23,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994983573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994983573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,582 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:23,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994983573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,582 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:23,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994983573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994983793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994983793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:23,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:23,935 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/2bd5920451174084adc6de4c5957e64f 2024-11-07T15:55:23,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/6c15c8ed034f456bbc1867cdc3964f68 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68 2024-11-07T15:55:23,942 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68, entries=100, sequenceid=163, filesize=9.5 K 2024-11-07T15:55:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2723c98127a449258060d5583deb5012 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012 2024-11-07T15:55:23,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012, entries=100, sequenceid=163, filesize=9.5 K 2024-11-07T15:55:23,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/2bd5920451174084adc6de4c5957e64f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f 2024-11-07T15:55:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f, entries=100, sequenceid=163, filesize=9.5 K 2024-11-07T15:55:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=174.43 KB/178620 for 8c07d911036021999ccdca9d657aee9b in 1252ms, sequenceid=163, compaction requested=false 2024-11-07T15:55:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-07T15:55:23,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-07T15:55:23,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-07T15:55:23,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8660 sec 2024-11-07T15:55:23,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.8700 sec 2024-11-07T15:55:24,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:24,108 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:24,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-07T15:55:24,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/689c38953d26486a971277018799cacf is 50, key is test_row_0/A:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:24,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742282_1458 (size=14541) 2024-11-07T15:55:24,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994984122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994984123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-07T15:55:24,233 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:24,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-07T15:55:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:24,235 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:24,236 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:24,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:24,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994984232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994984233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,240 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:24,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:55:24,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994984443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994984443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/689c38953d26486a971277018799cacf 2024-11-07T15:55:24,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17fc36d3b6542f6a81d407c87f5e161 is 50, key is test_row_0/B:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:24,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742283_1459 (size=12151) 2024-11-07T15:55:24,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17fc36d3b6542f6a81d407c87f5e161 2024-11-07T15:55:24,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/93d2743b7d9443a291ffc80b1d95f942 is 50, key is test_row_0/C:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:24,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:55:24,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742284_1460 (size=12151) 2024-11-07T15:55:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:24,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:55:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994984753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994984753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:24,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:24,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:24,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:55:24,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:24,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:24,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/93d2743b7d9443a291ffc80b1d95f942 2024-11-07T15:55:24,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/689c38953d26486a971277018799cacf as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf 2024-11-07T15:55:24,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf, entries=200, sequenceid=195, filesize=14.2 K 2024-11-07T15:55:24,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17fc36d3b6542f6a81d407c87f5e161 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161 2024-11-07T15:55:24,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T15:55:24,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/93d2743b7d9443a291ffc80b1d95f942 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942 2024-11-07T15:55:24,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942, entries=150, sequenceid=195, filesize=11.9 K 2024-11-07T15:55:24,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for 8c07d911036021999ccdca9d657aee9b in 855ms, sequenceid=195, compaction requested=true 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:24,963 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:24,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:24,963 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:24,964 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36689 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:24,964 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:24,964 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:24,964 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:24,964 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,964 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,965 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ab40536e6c7f4503ae993c7e462e50f7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.5 K 2024-11-07T15:55:24,965 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/1bc30153d8674a439df3ba0006eb585a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.8 K 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ab40536e6c7f4503ae993c7e462e50f7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc30153d8674a439df3ba0006eb585a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2723c98127a449258060d5583deb5012, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730994921391 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c15c8ed034f456bbc1867cdc3964f68, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730994921391 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting c17fc36d3b6542f6a81d407c87f5e161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:24,966 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 689c38953d26486a971277018799cacf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:24,974 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:24,974 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/5cb0e613636244788cc0bfbfe1fe34d0 is 50, key is test_row_0/B:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:24,975 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:24,976 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/caf2a8334d8e4c6d8c5ff392684a8372 is 50, key is test_row_0/A:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:24,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742285_1461 (size=12493) 2024-11-07T15:55:24,985 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/5cb0e613636244788cc0bfbfe1fe34d0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5cb0e613636244788cc0bfbfe1fe34d0 2024-11-07T15:55:24,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742286_1462 (size=12493) 2024-11-07T15:55:24,989 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 5cb0e613636244788cc0bfbfe1fe34d0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:24,989 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:24,989 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=13, startTime=1730994924963; duration=0sec 2024-11-07T15:55:24,989 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:24,989 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:24,989 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:24,990 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:24,990 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:24,991 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:24,991 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ff728eb574e94256a8e7b470cf2a1ee2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.5 K 2024-11-07T15:55:24,991 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ff728eb574e94256a8e7b470cf2a1ee2, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1730994920193 2024-11-07T15:55:24,991 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bd5920451174084adc6de4c5957e64f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1730994921391 2024-11-07T15:55:24,991 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 93d2743b7d9443a291ffc80b1d95f942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:24,998 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:24,998 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e97f114c2b41410299cdd94c75e87de9 is 50, key is test_row_0/C:col10/1730994924106/Put/seqid=0 2024-11-07T15:55:25,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742287_1463 (size=12493) 2024-11-07T15:55:25,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-07T15:55:25,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:25,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:55:25,006 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e97f114c2b41410299cdd94c75e87de9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e97f114c2b41410299cdd94c75e87de9 2024-11-07T15:55:25,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 is 50, key is test_row_0/A:col10/1730994924119/Put/seqid=0 2024-11-07T15:55:25,011 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into e97f114c2b41410299cdd94c75e87de9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:25,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:25,011 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=13, startTime=1730994924963; duration=0sec 2024-11-07T15:55:25,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:25,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:25,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742288_1464 (size=9757) 2024-11-07T15:55:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:25,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:25,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994985362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994985373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,391 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/caf2a8334d8e4c6d8c5ff392684a8372 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/caf2a8334d8e4c6d8c5ff392684a8372 2024-11-07T15:55:25,395 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into caf2a8334d8e4c6d8c5ff392684a8372(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:25,395 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:25,395 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=13, startTime=1730994924963; duration=0sec 2024-11-07T15:55:25,395 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:25,395 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:25,416 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 2024-11-07T15:55:25,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/99d115bd26ed4ed49ba095e3bf3b34c6 is 50, key is test_row_0/B:col10/1730994924119/Put/seqid=0 2024-11-07T15:55:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742289_1465 (size=9757) 2024-11-07T15:55:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994985482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994985493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994985593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994985593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,600 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,601 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994985613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,619 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:25,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994985693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994985703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:25,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:25,826 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/99d115bd26ed4ed49ba095e3bf3b34c6 2024-11-07T15:55:25,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/5a61f42b7136402386812a298d1065f5 is 50, key is test_row_0/C:col10/1730994924119/Put/seqid=0 2024-11-07T15:55:25,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742290_1466 (size=9757) 2024-11-07T15:55:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994986003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994986013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,236 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/5a61f42b7136402386812a298d1065f5 2024-11-07T15:55:26,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 2024-11-07T15:55:26,244 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4, entries=100, sequenceid=202, filesize=9.5 K 2024-11-07T15:55:26,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/99d115bd26ed4ed49ba095e3bf3b34c6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6 2024-11-07T15:55:26,248 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6, entries=100, sequenceid=202, filesize=9.5 K 2024-11-07T15:55:26,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/5a61f42b7136402386812a298d1065f5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5 2024-11-07T15:55:26,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5, entries=100, sequenceid=202, filesize=9.5 K 2024-11-07T15:55:26,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for 8c07d911036021999ccdca9d657aee9b in 1249ms, sequenceid=202, compaction requested=false 2024-11-07T15:55:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-07T15:55:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-07T15:55:26,255 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-07T15:55:26,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-11-07T15:55:26,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.0230 sec 2024-11-07T15:55:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-07T15:55:26,383 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:26,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-07T15:55:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:26,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:26,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:26,387 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:26,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:26,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-07T15:55:26,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a17995544edc4642986b7f356fce4c01 is 50, key is test_row_0/A:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:26,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742291_1467 (size=14541) 2024-11-07T15:55:26,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994986531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994986533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:26,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994986643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:26,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994986853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:26,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:26,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a17995544edc4642986b7f356fce4c01 2024-11-07T15:55:26,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/22e3814969b34c4984f7e3f0b5d78310 is 50, key is test_row_0/B:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:26,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742292_1468 (size=12151) 2024-11-07T15:55:26,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:26,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:26,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:27,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994987162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:27,167 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:27,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/22e3814969b34c4984f7e3f0b5d78310 2024-11-07T15:55:27,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/af08f8a07c42490799e404c347d633d4 is 50, key is test_row_0/C:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:27,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742293_1469 (size=12151) 2024-11-07T15:55:27,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:27,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994987543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:27,549 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:27,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994987672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:27,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/af08f8a07c42490799e404c347d633d4 2024-11-07T15:55:27,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/a17995544edc4642986b7f356fce4c01 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01 2024-11-07T15:55:27,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01, entries=200, sequenceid=235, filesize=14.2 K 2024-11-07T15:55:27,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/22e3814969b34c4984f7e3f0b5d78310 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310 2024-11-07T15:55:27,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:27,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:27,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T15:55:27,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/af08f8a07c42490799e404c347d633d4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4 2024-11-07T15:55:27,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4, entries=150, sequenceid=235, filesize=11.9 K 2024-11-07T15:55:27,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for 8c07d911036021999ccdca9d657aee9b in 1252ms, sequenceid=235, compaction requested=true 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:27,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:27,770 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:27,770 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:27,771 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:27,771 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:27,771 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:27,771 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:27,771 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,771 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,771 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/caf2a8334d8e4c6d8c5ff392684a8372, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=35.9 K 2024-11-07T15:55:27,771 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5cb0e613636244788cc0bfbfe1fe34d0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.6 K 2024-11-07T15:55:27,771 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb0e613636244788cc0bfbfe1fe34d0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:27,772 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf2a8334d8e4c6d8c5ff392684a8372, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:27,772 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 99d115bd26ed4ed49ba095e3bf3b34c6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730994924109 2024-11-07T15:55:27,772 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3b3b8b0adcc4e5ba66a0eb3f870f9d4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730994924109 2024-11-07T15:55:27,772 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 22e3814969b34c4984f7e3f0b5d78310, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:27,772 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a17995544edc4642986b7f356fce4c01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:27,779 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:27,780 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/29b83ff4a3964f3ab23445787e11d278 is 50, key is test_row_0/B:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:27,780 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:27,780 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/644f133a4be340eeb2afd1bdd05778b0 is 50, key is test_row_0/A:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:27,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742294_1470 (size=12595) 2024-11-07T15:55:27,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742295_1471 (size=12595) 2024-11-07T15:55:27,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-07T15:55:27,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:27,917 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-07T15:55:27,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d8d1d391136245d5a742e92a3607c16b is 50, key is test_row_0/A:col10/1730994926529/Put/seqid=0 2024-11-07T15:55:27,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742296_1472 (size=9757) 2024-11-07T15:55:27,931 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d8d1d391136245d5a742e92a3607c16b 2024-11-07T15:55:27,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/ea0c58d2a1c444c0825d61b49cf29650 is 50, key is test_row_0/B:col10/1730994926529/Put/seqid=0 2024-11-07T15:55:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742297_1473 (size=9757) 2024-11-07T15:55:28,189 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/29b83ff4a3964f3ab23445787e11d278 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/29b83ff4a3964f3ab23445787e11d278 2024-11-07T15:55:28,193 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 29b83ff4a3964f3ab23445787e11d278(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:28,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:28,193 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=13, startTime=1730994927770; duration=0sec 2024-11-07T15:55:28,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:28,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:28,193 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:28,196 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:28,197 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:28,197 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:28,197 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e97f114c2b41410299cdd94c75e87de9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.6 K 2024-11-07T15:55:28,197 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting e97f114c2b41410299cdd94c75e87de9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1730994923447 2024-11-07T15:55:28,199 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a61f42b7136402386812a298d1065f5, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1730994924109 2024-11-07T15:55:28,200 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting af08f8a07c42490799e404c347d633d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:28,200 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/644f133a4be340eeb2afd1bdd05778b0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/644f133a4be340eeb2afd1bdd05778b0 2024-11-07T15:55:28,208 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into 644f133a4be340eeb2afd1bdd05778b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:28,208 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:28,208 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=13, startTime=1730994927770; duration=0sec 2024-11-07T15:55:28,208 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:28,208 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:28,220 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:28,220 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/67dab930008c48d29553f43cd82ba749 is 50, key is test_row_0/C:col10/1730994926517/Put/seqid=0 2024-11-07T15:55:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742298_1474 (size=12595) 2024-11-07T15:55:28,231 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/67dab930008c48d29553f43cd82ba749 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/67dab930008c48d29553f43cd82ba749 2024-11-07T15:55:28,236 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into 67dab930008c48d29553f43cd82ba749(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:28,236 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:28,237 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=13, startTime=1730994927770; duration=0sec 2024-11-07T15:55:28,237 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:28,237 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:28,344 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/ea0c58d2a1c444c0825d61b49cf29650 2024-11-07T15:55:28,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/597163c223694d258d35fbbae2843ae6 is 50, key is test_row_0/C:col10/1730994926529/Put/seqid=0 2024-11-07T15:55:28,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742299_1475 (size=9757) 2024-11-07T15:55:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:28,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:28,756 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/597163c223694d258d35fbbae2843ae6 2024-11-07T15:55:28,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/d8d1d391136245d5a742e92a3607c16b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b 2024-11-07T15:55:28,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b, entries=100, sequenceid=240, filesize=9.5 K 2024-11-07T15:55:28,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/ea0c58d2a1c444c0825d61b49cf29650 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650 2024-11-07T15:55:28,769 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650, entries=100, sequenceid=240, filesize=9.5 K 2024-11-07T15:55:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/597163c223694d258d35fbbae2843ae6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6 2024-11-07T15:55:28,773 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6, entries=100, sequenceid=240, filesize=9.5 K 2024-11-07T15:55:28,774 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=100.63 KB/103050 for 8c07d911036021999ccdca9d657aee9b in 858ms, sequenceid=240, compaction requested=false 2024-11-07T15:55:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:28,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-07T15:55:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-07T15:55:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:28,779 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:28,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:55:28,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-07T15:55:28,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3880 sec 2024-11-07T15:55:28,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/b9741303092d42208cdcf756a696d77c is 50, key is test_row_0/A:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:28,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.3960 sec 2024-11-07T15:55:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742300_1476 (size=14691) 2024-11-07T15:55:28,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/b9741303092d42208cdcf756a696d77c 2024-11-07T15:55:28,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2cab2c287770474b9d6c4dc278e74bc6 is 50, key is test_row_0/B:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:28,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742301_1477 (size=12251) 2024-11-07T15:55:28,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994988890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:28,896 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994989003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2cab2c287770474b9d6c4dc278e74bc6 2024-11-07T15:55:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994989213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/21f98ff3bc2d40eaa60ae54d3f318bf7 is 50, key is test_row_0/C:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:29,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742302_1478 (size=12251) 2024-11-07T15:55:29,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994989533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994989562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,572 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/21f98ff3bc2d40eaa60ae54d3f318bf7 2024-11-07T15:55:29,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/b9741303092d42208cdcf756a696d77c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c 2024-11-07T15:55:29,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33112 deadline: 1730994989623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33132 deadline: 1730994989623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,630 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,631 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c, entries=200, sequenceid=262, filesize=14.3 K 2024-11-07T15:55:29,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/2cab2c287770474b9d6c4dc278e74bc6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6 2024-11-07T15:55:29,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6, entries=150, sequenceid=262, filesize=12.0 K 2024-11-07T15:55:29,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/21f98ff3bc2d40eaa60ae54d3f318bf7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7 2024-11-07T15:55:29,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7, entries=150, sequenceid=262, filesize=12.0 K 2024-11-07T15:55:29,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 8c07d911036021999ccdca9d657aee9b in 862ms, sequenceid=262, compaction requested=true 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:29,641 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-07T15:55:29,642 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:29,642 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:29,642 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:29,642 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:29,642 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/644f133a4be340eeb2afd1bdd05778b0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=36.2 K 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:29,643 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644f133a4be340eeb2afd1bdd05778b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:29,643 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/29b83ff4a3964f3ab23445787e11d278, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.8 K 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d1d391136245d5a742e92a3607c16b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994926527 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 29b83ff4a3964f3ab23445787e11d278, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9741303092d42208cdcf756a696d77c, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:29,643 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ea0c58d2a1c444c0825d61b49cf29650, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994926527 2024-11-07T15:55:29,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:29,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:55:29,644 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cab2c287770474b9d6c4dc278e74bc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:29,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/eae372ca7eb249f290b29e1343097a19 is 50, key is test_row_0/A:col10/1730994928877/Put/seqid=0 2024-11-07T15:55:29,652 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:29,652 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:29,653 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/dc0e41dc3b6748cd8ad47b23124e0fd6 is 50, key is test_row_0/A:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:29,653 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/1d50ac17eca64d8b8f345b4fa9a0b8c6 is 50, key is test_row_0/B:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:29,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742303_1479 (size=14741) 2024-11-07T15:55:29,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/eae372ca7eb249f290b29e1343097a19 2024-11-07T15:55:29,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742305_1481 (size=12797) 2024-11-07T15:55:29,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/5d7d7a71ea70471d9d34257a2a23f25d is 50, key is test_row_0/B:col10/1730994928877/Put/seqid=0 2024-11-07T15:55:29,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742304_1480 (size=12797) 2024-11-07T15:55:29,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742306_1482 (size=12301) 2024-11-07T15:55:29,685 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/dc0e41dc3b6748cd8ad47b23124e0fd6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dc0e41dc3b6748cd8ad47b23124e0fd6 2024-11-07T15:55:29,688 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into dc0e41dc3b6748cd8ad47b23124e0fd6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:29,689 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=13, startTime=1730994929641; duration=0sec 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:29,689 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:29,690 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:29,690 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/67dab930008c48d29553f43cd82ba749, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=33.8 K 2024-11-07T15:55:29,690 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67dab930008c48d29553f43cd82ba749, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1730994925355 2024-11-07T15:55:29,690 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 597163c223694d258d35fbbae2843ae6, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1730994926527 2024-11-07T15:55:29,690 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21f98ff3bc2d40eaa60ae54d3f318bf7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:29,696 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:29,696 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e9d806a666de4e6a9ba608a2091f86d8 is 50, key is test_row_0/C:col10/1730994928776/Put/seqid=0 2024-11-07T15:55:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742307_1483 (size=12797) 2024-11-07T15:55:29,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994989793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:29,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:29,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994989903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:30,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994990053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:30,081 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/1d50ac17eca64d8b8f345b4fa9a0b8c6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/1d50ac17eca64d8b8f345b4fa9a0b8c6 2024-11-07T15:55:30,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/5d7d7a71ea70471d9d34257a2a23f25d 2024-11-07T15:55:30,085 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 1d50ac17eca64d8b8f345b4fa9a0b8c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:30,085 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:30,085 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=13, startTime=1730994929641; duration=0sec 2024-11-07T15:55:30,085 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:30,085 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:30,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/45efdf4d726646e99611a53d7ab78f33 is 50, key is test_row_0/C:col10/1730994928877/Put/seqid=0 2024-11-07T15:55:30,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742308_1484 (size=12301) 2024-11-07T15:55:30,104 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e9d806a666de4e6a9ba608a2091f86d8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9d806a666de4e6a9ba608a2091f86d8 2024-11-07T15:55:30,108 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into e9d806a666de4e6a9ba608a2091f86d8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:30,108 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:30,109 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=13, startTime=1730994929641; duration=0sec 2024-11-07T15:55:30,109 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:30,109 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:30,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:30,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994990112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:30,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994990423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:30,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:30,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/45efdf4d726646e99611a53d7ab78f33 2024-11-07T15:55:30,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/eae372ca7eb249f290b29e1343097a19 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19 2024-11-07T15:55:30,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19, entries=200, sequenceid=280, filesize=14.4 K 2024-11-07T15:55:30,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/5d7d7a71ea70471d9d34257a2a23f25d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d 2024-11-07T15:55:30,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T15:55:30,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/45efdf4d726646e99611a53d7ab78f33 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33 2024-11-07T15:55:30,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33, entries=150, sequenceid=280, filesize=12.0 K 2024-11-07T15:55:30,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8c07d911036021999ccdca9d657aee9b in 869ms, sequenceid=280, compaction requested=false 2024-11-07T15:55:30,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-07T15:55:30,563 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:30,564 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-07T15:55:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:55:30,566 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:30,567 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:30,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:55:30,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-07T15:55:30,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:30,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:55:30,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/20faaa7fec36403cb7d817e5a10b2b10 is 50, key is test_row_0/A:col10/1730994929706/Put/seqid=0 2024-11-07T15:55:30,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742309_1485 (size=12301) 2024-11-07T15:55:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:55:30,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:30,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:31,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994991063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,075 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,075 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:31,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994991065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:31,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/20faaa7fec36403cb7d817e5a10b2b10 2024-11-07T15:55:31,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/045954e49dd74a7fb788abd2e6329886 is 50, key is test_row_0/B:col10/1730994929706/Put/seqid=0 2024-11-07T15:55:31,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742310_1486 (size=12301) 2024-11-07T15:55:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:55:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994991183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:31,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994991403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:31,546 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/045954e49dd74a7fb788abd2e6329886 2024-11-07T15:55:31,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/128f35bea2804b8da01e9b0c55c963d0 is 50, key is test_row_0/C:col10/1730994929706/Put/seqid=0 2024-11-07T15:55:31,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742311_1487 (size=12301) 2024-11-07T15:55:31,565 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/128f35bea2804b8da01e9b0c55c963d0 2024-11-07T15:55:31,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/20faaa7fec36403cb7d817e5a10b2b10 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10 2024-11-07T15:55:31,572 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:55:31,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/045954e49dd74a7fb788abd2e6329886 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886 2024-11-07T15:55:31,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:55:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/128f35bea2804b8da01e9b0c55c963d0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0 2024-11-07T15:55:31,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0, entries=150, sequenceid=301, filesize=12.0 K 2024-11-07T15:55:31,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8c07d911036021999ccdca9d657aee9b in 861ms, sequenceid=301, compaction requested=true 2024-11-07T15:55:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:31,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-07T15:55:31,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-07T15:55:31,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-07T15:55:31,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0150 sec 2024-11-07T15:55:31,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0200 sec 2024-11-07T15:55:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-07T15:55:31,704 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:31,704 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:31,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-07T15:55:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:31,706 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:31,707 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:31,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:31,720 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:31,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:55:31,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/51a05e43a40c4ed99275afcd45ec3734 is 50, key is test_row_0/A:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742312_1488 (size=14741) 2024-11-07T15:55:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:31,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:31,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:31,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:31,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:31,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:31,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994991862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:31,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994991973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:31,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:32,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:32,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/51a05e43a40c4ed99275afcd45ec3734 2024-11-07T15:55:32,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/015c0fe47bc34dca940322c378e28699 is 50, key is test_row_0/B:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:32,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742313_1489 (size=12301) 2024-11-07T15:55:32,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,166 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994992183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:32,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:32,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:32,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:32,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994992502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:32,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:32,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/015c0fe47bc34dca940322c378e28699 2024-11-07T15:55:32,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/25cbe3f47ba64d1abf473cb5935155c8 is 50, key is test_row_0/C:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:32,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742314_1490 (size=12301) 2024-11-07T15:55:32,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:32,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:32,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/25cbe3f47ba64d1abf473cb5935155c8 2024-11-07T15:55:32,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/51a05e43a40c4ed99275afcd45ec3734 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734 2024-11-07T15:55:32,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734, entries=200, sequenceid=320, filesize=14.4 K 2024-11-07T15:55:32,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/015c0fe47bc34dca940322c378e28699 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699 2024-11-07T15:55:32,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699, entries=150, sequenceid=320, filesize=12.0 K 2024-11-07T15:55:32,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/25cbe3f47ba64d1abf473cb5935155c8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8 2024-11-07T15:55:32,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-07T15:55:32,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 8c07d911036021999ccdca9d657aee9b in 1261ms, sequenceid=320, compaction requested=true 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8c07d911036021999ccdca9d657aee9b:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:32,981 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:32,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:32,981 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:32,982 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:32,982 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:32,982 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/A is initiating minor compaction (all files) 2024-11-07T15:55:32,982 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/B is initiating minor compaction (all files) 2024-11-07T15:55:32,982 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/A in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,982 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/B in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:32,982 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dc0e41dc3b6748cd8ad47b23124e0fd6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=53.3 K 2024-11-07T15:55:32,982 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/1d50ac17eca64d8b8f345b4fa9a0b8c6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=48.5 K 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc0e41dc3b6748cd8ad47b23124e0fd6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d50ac17eca64d8b8f345b4fa9a0b8c6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting eae372ca7eb249f290b29e1343097a19, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994928866 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d7d7a71ea70471d9d34257a2a23f25d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994928866 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20faaa7fec36403cb7d817e5a10b2b10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730994929706 2024-11-07T15:55:32,983 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 045954e49dd74a7fb788abd2e6329886, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730994929706 2024-11-07T15:55:32,984 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51a05e43a40c4ed99275afcd45ec3734, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730994931049 2024-11-07T15:55:32,984 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 015c0fe47bc34dca940322c378e28699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730994931053 2024-11-07T15:55:32,990 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:32,991 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/0c1cf0b3eb4f46aebfee8d5e4f654622 is 50, key is test_row_0/A:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:32,991 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#B#compaction#415 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:32,992 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/94429e3b2c9a43e99f4fc62bd725641f is 50, key is test_row_0/B:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:32,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742315_1491 (size=12983) 2024-11-07T15:55:32,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742316_1492 (size=12983) 2024-11-07T15:55:33,003 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/0c1cf0b3eb4f46aebfee8d5e4f654622 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/0c1cf0b3eb4f46aebfee8d5e4f654622 2024-11-07T15:55:33,005 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/94429e3b2c9a43e99f4fc62bd725641f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/94429e3b2c9a43e99f4fc62bd725641f 2024-11-07T15:55:33,007 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/A of 8c07d911036021999ccdca9d657aee9b into 0c1cf0b3eb4f46aebfee8d5e4f654622(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:33,007 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:33,007 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/A, priority=12, startTime=1730994932981; duration=0sec 2024-11-07T15:55:33,007 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:33,007 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:A 2024-11-07T15:55:33,007 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:33,008 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49700 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:33,008 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 8c07d911036021999ccdca9d657aee9b/C is initiating minor compaction (all files) 2024-11-07T15:55:33,008 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8c07d911036021999ccdca9d657aee9b/C in TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,008 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9d806a666de4e6a9ba608a2091f86d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp, totalSize=48.5 K 2024-11-07T15:55:33,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9d806a666de4e6a9ba608a2091f86d8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1730994928736 2024-11-07T15:55:33,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45efdf4d726646e99611a53d7ab78f33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1730994928866 2024-11-07T15:55:33,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128f35bea2804b8da01e9b0c55c963d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1730994929706 2024-11-07T15:55:33,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25cbe3f47ba64d1abf473cb5935155c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1730994931053 2024-11-07T15:55:33,011 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/B of 8c07d911036021999ccdca9d657aee9b into 94429e3b2c9a43e99f4fc62bd725641f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:33,011 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:33,011 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/B, priority=12, startTime=1730994932981; duration=0sec 2024-11-07T15:55:33,012 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:33,012 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:B 2024-11-07T15:55:33,019 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8c07d911036021999ccdca9d657aee9b#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:33,019 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/2b44fb4b96d743d4b2d65e25f7edabf0 is 50, key is test_row_0/C:col10/1730994931053/Put/seqid=0 2024-11-07T15:55:33,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:33,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:33,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:55:33,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742317_1493 (size=12983) 2024-11-07T15:55:33,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/f63dba95455649cd83be7dd81812e7c2 is 50, key is test_row_0/A:col10/1730994933019/Put/seqid=0 2024-11-07T15:55:33,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742318_1494 (size=14741) 2024-11-07T15:55:33,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/f63dba95455649cd83be7dd81812e7c2 2024-11-07T15:55:33,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17cf0e8b0b24ca3afbd80903a23fe72 is 50, key is test_row_0/B:col10/1730994933019/Put/seqid=0 2024-11-07T15:55:33,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742319_1495 (size=12301) 2024-11-07T15:55:33,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994993127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994993129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994993243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994993243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,427 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/2b44fb4b96d743d4b2d65e25f7edabf0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2b44fb4b96d743d4b2d65e25f7edabf0 2024-11-07T15:55:33,431 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8c07d911036021999ccdca9d657aee9b/C of 8c07d911036021999ccdca9d657aee9b into 2b44fb4b96d743d4b2d65e25f7edabf0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:33,431 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:33,431 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., storeName=8c07d911036021999ccdca9d657aee9b/C, priority=12, startTime=1730994932981; duration=0sec 2024-11-07T15:55:33,431 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:33,431 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8c07d911036021999ccdca9d657aee9b:C 2024-11-07T15:55:33,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17cf0e8b0b24ca3afbd80903a23fe72 2024-11-07T15:55:33,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e67c3261265748bd9df9a78a922085d9 is 50, key is test_row_0/C:col10/1730994933019/Put/seqid=0 2024-11-07T15:55:33,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742320_1496 (size=12301) 2024-11-07T15:55:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994993453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994993453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,607 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-11-07T15:55:33,607 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:33,607 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,607 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:33,608 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-11-07T15:55:33,608 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:33,608 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,608 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:33,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33110 deadline: 1730994993603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,613 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 8c07d911036021999ccdca9d657aee9b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,614 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-07T15:55:33,614 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:33,614 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,614 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:33,618 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-07T15:55:33,618 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:33,618 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,618 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:33,619 INFO [Thread-1912 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1912. 2024-11-07T15:55:33,619 DEBUG [Thread-1912 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:33,619 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,619 DEBUG [Thread-1912 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:33,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:33,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33146 deadline: 1730994993763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33096 deadline: 1730994993763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 , the old value is region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., hostname=c2682f91c1e2,42309,1730994796599, seqNum=2 because the exception is null or not the one we care about 2024-11-07T15:55:33,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:33,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:33,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e67c3261265748bd9df9a78a922085d9 2024-11-07T15:55:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:33,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/f63dba95455649cd83be7dd81812e7c2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/f63dba95455649cd83be7dd81812e7c2 2024-11-07T15:55:33,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/f63dba95455649cd83be7dd81812e7c2, entries=200, sequenceid=340, filesize=14.4 K 2024-11-07T15:55:33,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c17cf0e8b0b24ca3afbd80903a23fe72 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17cf0e8b0b24ca3afbd80903a23fe72 2024-11-07T15:55:33,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17cf0e8b0b24ca3afbd80903a23fe72, entries=150, sequenceid=340, filesize=12.0 K 2024-11-07T15:55:33,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/e67c3261265748bd9df9a78a922085d9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e67c3261265748bd9df9a78a922085d9 2024-11-07T15:55:33,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e67c3261265748bd9df9a78a922085d9, entries=150, sequenceid=340, filesize=12.0 K 2024-11-07T15:55:33,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8c07d911036021999ccdca9d657aee9b in 845ms, sequenceid=340, compaction requested=false 2024-11-07T15:55:33,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:34,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-07T15:55:34,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:34,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:55:34,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/484b8a7ae6ef490881707886e2aac6ac is 50, key is test_row_0/A:col10/1730994933112/Put/seqid=0 2024-11-07T15:55:34,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742321_1497 (size=12301) 2024-11-07T15:55:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:34,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. as already flushing 2024-11-07T15:55:34,275 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-07T15:55:34,275 INFO [Thread-1909 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1909. 2024-11-07T15:55:34,275 DEBUG [Thread-1909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:34,275 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:34,275 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:34,275 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:34,275 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:34,275 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:34,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:34,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:34,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/484b8a7ae6ef490881707886e2aac6ac 2024-11-07T15:55:34,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c96305410b8645489bc3956096d59ca0 is 50, key is test_row_0/B:col10/1730994933112/Put/seqid=0 2024-11-07T15:55:34,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742322_1498 (size=12301) 2024-11-07T15:55:34,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c96305410b8645489bc3956096d59ca0 2024-11-07T15:55:34,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/305a722bc0294e4387fe2a7a8dade09d is 50, key is test_row_0/C:col10/1730994933112/Put/seqid=0 2024-11-07T15:55:34,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742323_1499 (size=12301) 2024-11-07T15:55:35,227 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/305a722bc0294e4387fe2a7a8dade09d 2024-11-07T15:55:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/484b8a7ae6ef490881707886e2aac6ac as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/484b8a7ae6ef490881707886e2aac6ac 2024-11-07T15:55:35,234 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/484b8a7ae6ef490881707886e2aac6ac, entries=150, sequenceid=359, filesize=12.0 K 2024-11-07T15:55:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/c96305410b8645489bc3956096d59ca0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c96305410b8645489bc3956096d59ca0 2024-11-07T15:55:35,237 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c96305410b8645489bc3956096d59ca0, entries=150, sequenceid=359, filesize=12.0 K 2024-11-07T15:55:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/305a722bc0294e4387fe2a7a8dade09d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/305a722bc0294e4387fe2a7a8dade09d 2024-11-07T15:55:35,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/305a722bc0294e4387fe2a7a8dade09d, entries=150, sequenceid=359, filesize=12.0 K 2024-11-07T15:55:35,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 8c07d911036021999ccdca9d657aee9b in 1240ms, sequenceid=359, compaction requested=true 2024-11-07T15:55:35,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 8c07d911036021999ccdca9d657aee9b: 2024-11-07T15:55:35,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:35,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-07T15:55:35,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-07T15:55:35,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-07T15:55:35,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5360 sec 2024-11-07T15:55:35,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 3.5400 sec 2024-11-07T15:55:35,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-07T15:55:35,873 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:39,675 INFO [Thread-1901 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1901. 2024-11-07T15:55:39,675 DEBUG [Thread-1901 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:39,675 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:39,675 DEBUG [Thread-1901 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:39,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:39,704 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-11-07T15:55:39,704 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:39,704 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:39,705 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:39,705 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:43,675 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-11-07T15:55:43,675 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:55:43,675 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:43,675 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:43,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 81 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1588 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4763 rows 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1588 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4763 rows 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1585 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4754 rows 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1585 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4750 rows 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1590 2024-11-07T15:55:43,675 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4769 rows 2024-11-07T15:55:43,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:55:43,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:55:43,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:43,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:43,676 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:55:43,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:55:43,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:43,679 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994943679"}]},"ts":"1730994943679"} 2024-11-07T15:55:43,680 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:55:43,680 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:55:43,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:55:43,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, UNASSIGN}] 2024-11-07T15:55:43,683 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, UNASSIGN 2024-11-07T15:55:43,684 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8c07d911036021999ccdca9d657aee9b, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:43,685 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, UNASSIGN because future has completed 2024-11-07T15:55:43,685 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:55:43,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:43,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:43,837 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:43,837 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:55:43,837 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 8c07d911036021999ccdca9d657aee9b, disabling compactions & flushes 2024-11-07T15:55:43,837 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:43,837 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:43,837 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. after waiting 0 ms 2024-11-07T15:55:43,837 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:43,838 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 8c07d911036021999ccdca9d657aee9b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-07T15:55:43,841 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/dd1e32bde98f412a8040f330210cab3d is 50, key is test_row_0/A:col10/1730994939703/Put/seqid=0 2024-11-07T15:55:43,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742324_1500 (size=9857) 2024-11-07T15:55:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:44,246 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/dd1e32bde98f412a8040f330210cab3d 2024-11-07T15:55:44,253 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/98e7e8ea26744fb5a2febefb377e1b31 is 50, key is test_row_0/B:col10/1730994939703/Put/seqid=0 2024-11-07T15:55:44,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742325_1501 (size=9857) 2024-11-07T15:55:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:44,657 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/98e7e8ea26744fb5a2febefb377e1b31 2024-11-07T15:55:44,663 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/75725ceb5a9145b181cd5d9ec83ce540 is 50, key is test_row_0/C:col10/1730994939703/Put/seqid=0 2024-11-07T15:55:44,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742326_1502 (size=9857) 2024-11-07T15:55:44,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:44,926 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:55:45,068 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/75725ceb5a9145b181cd5d9ec83ce540 2024-11-07T15:55:45,075 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/A/dd1e32bde98f412a8040f330210cab3d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dd1e32bde98f412a8040f330210cab3d 2024-11-07T15:55:45,078 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dd1e32bde98f412a8040f330210cab3d, entries=100, sequenceid=367, filesize=9.6 K 2024-11-07T15:55:45,079 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/B/98e7e8ea26744fb5a2febefb377e1b31 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/98e7e8ea26744fb5a2febefb377e1b31 2024-11-07T15:55:45,082 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/98e7e8ea26744fb5a2febefb377e1b31, entries=100, sequenceid=367, filesize=9.6 K 2024-11-07T15:55:45,083 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/.tmp/C/75725ceb5a9145b181cd5d9ec83ce540 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/75725ceb5a9145b181cd5d9ec83ce540 2024-11-07T15:55:45,086 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/75725ceb5a9145b181cd5d9ec83ce540, entries=100, sequenceid=367, filesize=9.6 K 2024-11-07T15:55:45,087 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8c07d911036021999ccdca9d657aee9b in 1250ms, sequenceid=367, compaction requested=true 2024-11-07T15:55:45,088 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/5015e6bfc8c040edb554a342aa6df681, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/fc09d5f126dd491883a3593f1697c172, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/1bc30153d8674a439df3ba0006eb585a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/caf2a8334d8e4c6d8c5ff392684a8372, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/644f133a4be340eeb2afd1bdd05778b0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dc0e41dc3b6748cd8ad47b23124e0fd6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734] to archive 2024-11-07T15:55:45,089 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:45,090 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/5015e6bfc8c040edb554a342aa6df681 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/5015e6bfc8c040edb554a342aa6df681 2024-11-07T15:55:45,091 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a822df6a58214dde896b10b2ca81c2fe 2024-11-07T15:55:45,093 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/74af0a9879cc4fdb8171549bba88a601 2024-11-07T15:55:45,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/aa295fa471f340d4b005d904940d30bc 2024-11-07T15:55:45,095 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/fc09d5f126dd491883a3593f1697c172 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/fc09d5f126dd491883a3593f1697c172 2024-11-07T15:55:45,096 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/e22b286b98ca4b9181f8755835771915 2024-11-07T15:55:45,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/1bc30153d8674a439df3ba0006eb585a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/1bc30153d8674a439df3ba0006eb585a 2024-11-07T15:55:45,098 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/706fce3e84644f2ca24439cdc52e0470 2024-11-07T15:55:45,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/6c15c8ed034f456bbc1867cdc3964f68 2024-11-07T15:55:45,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/689c38953d26486a971277018799cacf 2024-11-07T15:55:45,102 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/caf2a8334d8e4c6d8c5ff392684a8372 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/caf2a8334d8e4c6d8c5ff392684a8372 2024-11-07T15:55:45,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d3b3b8b0adcc4e5ba66a0eb3f870f9d4 2024-11-07T15:55:45,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/a17995544edc4642986b7f356fce4c01 2024-11-07T15:55:45,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/644f133a4be340eeb2afd1bdd05778b0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/644f133a4be340eeb2afd1bdd05778b0 2024-11-07T15:55:45,106 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/d8d1d391136245d5a742e92a3607c16b 2024-11-07T15:55:45,107 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/b9741303092d42208cdcf756a696d77c 2024-11-07T15:55:45,108 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dc0e41dc3b6748cd8ad47b23124e0fd6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dc0e41dc3b6748cd8ad47b23124e0fd6 2024-11-07T15:55:45,109 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/eae372ca7eb249f290b29e1343097a19 2024-11-07T15:55:45,110 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/20faaa7fec36403cb7d817e5a10b2b10 2024-11-07T15:55:45,111 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/51a05e43a40c4ed99275afcd45ec3734 2024-11-07T15:55:45,113 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/841e1cef500343eab551d7cfd4802d7b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/184bdcf36a934513b6e1c762e56033fc, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ab40536e6c7f4503ae993c7e462e50f7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5cb0e613636244788cc0bfbfe1fe34d0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/29b83ff4a3964f3ab23445787e11d278, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/1d50ac17eca64d8b8f345b4fa9a0b8c6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699] to archive 2024-11-07T15:55:45,113 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:45,115 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/841e1cef500343eab551d7cfd4802d7b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/841e1cef500343eab551d7cfd4802d7b 2024-11-07T15:55:45,116 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/cb1ccef920424d66abe5552906a9c023 2024-11-07T15:55:45,117 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/0cf4e1d4c28d43c6825fc3f86ef67ede 2024-11-07T15:55:45,118 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/184bdcf36a934513b6e1c762e56033fc to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/184bdcf36a934513b6e1c762e56033fc 2024-11-07T15:55:45,119 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/3328d15410ab49ec9c42c0420e31b99f 2024-11-07T15:55:45,120 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/563432b1caf542febfc7817aa9224865 2024-11-07T15:55:45,121 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ab40536e6c7f4503ae993c7e462e50f7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ab40536e6c7f4503ae993c7e462e50f7 2024-11-07T15:55:45,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/7741ece7693c49a79178c42e4f470068 2024-11-07T15:55:45,123 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2723c98127a449258060d5583deb5012 2024-11-07T15:55:45,123 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5cb0e613636244788cc0bfbfe1fe34d0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5cb0e613636244788cc0bfbfe1fe34d0 2024-11-07T15:55:45,126 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17fc36d3b6542f6a81d407c87f5e161 2024-11-07T15:55:45,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/99d115bd26ed4ed49ba095e3bf3b34c6 2024-11-07T15:55:45,128 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/29b83ff4a3964f3ab23445787e11d278 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/29b83ff4a3964f3ab23445787e11d278 2024-11-07T15:55:45,129 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/22e3814969b34c4984f7e3f0b5d78310 2024-11-07T15:55:45,130 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/ea0c58d2a1c444c0825d61b49cf29650 2024-11-07T15:55:45,131 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/1d50ac17eca64d8b8f345b4fa9a0b8c6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/1d50ac17eca64d8b8f345b4fa9a0b8c6 2024-11-07T15:55:45,132 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/2cab2c287770474b9d6c4dc278e74bc6 2024-11-07T15:55:45,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/5d7d7a71ea70471d9d34257a2a23f25d 2024-11-07T15:55:45,135 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/045954e49dd74a7fb788abd2e6329886 2024-11-07T15:55:45,136 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/015c0fe47bc34dca940322c378e28699 2024-11-07T15:55:45,137 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/826aba1b39ac47bfad509a05793ecfa7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/bcbbede10763432ea7bc094bc8bd880d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ff728eb574e94256a8e7b470cf2a1ee2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e97f114c2b41410299cdd94c75e87de9, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/67dab930008c48d29553f43cd82ba749, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9d806a666de4e6a9ba608a2091f86d8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8] to archive 2024-11-07T15:55:45,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:55:45,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/826aba1b39ac47bfad509a05793ecfa7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/826aba1b39ac47bfad509a05793ecfa7 2024-11-07T15:55:45,140 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/b2e548f589fc46c4bf8f5f150c295088 2024-11-07T15:55:45,141 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/16f68d073e9f44209799f07a9bb492d4 2024-11-07T15:55:45,142 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/bcbbede10763432ea7bc094bc8bd880d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/bcbbede10763432ea7bc094bc8bd880d 2024-11-07T15:55:45,143 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/10794648b8954b75b1cd98469cdded90 2024-11-07T15:55:45,144 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ecd8d0a6ebe24a639b87d785a23a705e 2024-11-07T15:55:45,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ff728eb574e94256a8e7b470cf2a1ee2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/ff728eb574e94256a8e7b470cf2a1ee2 2024-11-07T15:55:45,147 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9895e6f26e54de9b6e622ddf134f2db 2024-11-07T15:55:45,148 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2bd5920451174084adc6de4c5957e64f 2024-11-07T15:55:45,149 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e97f114c2b41410299cdd94c75e87de9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e97f114c2b41410299cdd94c75e87de9 2024-11-07T15:55:45,150 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/93d2743b7d9443a291ffc80b1d95f942 2024-11-07T15:55:45,151 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/5a61f42b7136402386812a298d1065f5 2024-11-07T15:55:45,152 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/67dab930008c48d29553f43cd82ba749 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/67dab930008c48d29553f43cd82ba749 2024-11-07T15:55:45,153 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/af08f8a07c42490799e404c347d633d4 2024-11-07T15:55:45,154 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/597163c223694d258d35fbbae2843ae6 2024-11-07T15:55:45,155 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9d806a666de4e6a9ba608a2091f86d8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e9d806a666de4e6a9ba608a2091f86d8 2024-11-07T15:55:45,157 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/21f98ff3bc2d40eaa60ae54d3f318bf7 2024-11-07T15:55:45,158 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/45efdf4d726646e99611a53d7ab78f33 2024-11-07T15:55:45,164 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/128f35bea2804b8da01e9b0c55c963d0 2024-11-07T15:55:45,165 DEBUG [StoreCloser-TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/25cbe3f47ba64d1abf473cb5935155c8 2024-11-07T15:55:45,169 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=1 2024-11-07T15:55:45,170 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. 2024-11-07T15:55:45,170 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 8c07d911036021999ccdca9d657aee9b: Waiting for close lock at 1730994943837Running coprocessor pre-close hooks at 1730994943837Disabling compacts and flushes for region at 1730994943837Disabling writes for close at 1730994943837Obtaining lock to block concurrent updates at 1730994943838 (+1 ms)Preparing flush snapshotting stores in 8c07d911036021999ccdca9d657aee9b at 1730994943838Finished memstore snapshotting TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1730994943838Flushing stores of TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b. at 1730994943838Flushing 8c07d911036021999ccdca9d657aee9b/A: creating writer at 1730994943838Flushing 8c07d911036021999ccdca9d657aee9b/A: appending metadata at 1730994943841 (+3 ms)Flushing 8c07d911036021999ccdca9d657aee9b/A: closing flushed file at 1730994943841Flushing 8c07d911036021999ccdca9d657aee9b/B: creating writer at 1730994944250 (+409 ms)Flushing 8c07d911036021999ccdca9d657aee9b/B: appending metadata at 1730994944253 (+3 ms)Flushing 8c07d911036021999ccdca9d657aee9b/B: closing flushed file at 1730994944253Flushing 8c07d911036021999ccdca9d657aee9b/C: creating writer at 1730994944660 (+407 ms)Flushing 8c07d911036021999ccdca9d657aee9b/C: appending metadata at 1730994944663 (+3 ms)Flushing 8c07d911036021999ccdca9d657aee9b/C: closing flushed file at 1730994944663Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4df5644: reopening flushed file at 1730994945074 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3815545b: reopening flushed file at 1730994945078 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2dd1c15e: reopening flushed file at 1730994945083 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8c07d911036021999ccdca9d657aee9b in 1250ms, sequenceid=367, compaction requested=true at 1730994945087 (+4 ms)Writing region close event to WAL at 1730994945166 (+79 ms)Running coprocessor post-close hooks at 1730994945170 (+4 ms)Closed at 1730994945170 2024-11-07T15:55:45,173 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:45,174 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=8c07d911036021999ccdca9d657aee9b, regionState=CLOSED 2024-11-07T15:55:45,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:45,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-07T15:55:45,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 8c07d911036021999ccdca9d657aee9b, server=c2682f91c1e2,42309,1730994796599 in 1.4910 sec 2024-11-07T15:55:45,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-07T15:55:45,181 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8c07d911036021999ccdca9d657aee9b, UNASSIGN in 1.4970 sec 2024-11-07T15:55:45,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-07T15:55:45,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-07T15:55:45,186 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994945185"}]},"ts":"1730994945185"} 2024-11-07T15:55:45,187 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:55:45,187 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:55:45,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-07T15:55:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-07T15:55:45,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:45,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:55:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,826 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T15:55:45,826 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,828 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:45,830 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/recovered.edits] 2024-11-07T15:55:45,831 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/0c1cf0b3eb4f46aebfee8d5e4f654622 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/0c1cf0b3eb4f46aebfee8d5e4f654622 2024-11-07T15:55:45,832 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/484b8a7ae6ef490881707886e2aac6ac to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/484b8a7ae6ef490881707886e2aac6ac 2024-11-07T15:55:45,833 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dd1e32bde98f412a8040f330210cab3d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/dd1e32bde98f412a8040f330210cab3d 2024-11-07T15:55:45,834 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/f63dba95455649cd83be7dd81812e7c2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/A/f63dba95455649cd83be7dd81812e7c2 2024-11-07T15:55:45,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/94429e3b2c9a43e99f4fc62bd725641f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/94429e3b2c9a43e99f4fc62bd725641f 2024-11-07T15:55:45,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/98e7e8ea26744fb5a2febefb377e1b31 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/98e7e8ea26744fb5a2febefb377e1b31 2024-11-07T15:55:45,836 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17cf0e8b0b24ca3afbd80903a23fe72 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c17cf0e8b0b24ca3afbd80903a23fe72 2024-11-07T15:55:45,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c96305410b8645489bc3956096d59ca0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/B/c96305410b8645489bc3956096d59ca0 2024-11-07T15:55:45,838 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2b44fb4b96d743d4b2d65e25f7edabf0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/2b44fb4b96d743d4b2d65e25f7edabf0 2024-11-07T15:55:45,839 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/305a722bc0294e4387fe2a7a8dade09d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/305a722bc0294e4387fe2a7a8dade09d 2024-11-07T15:55:45,840 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/75725ceb5a9145b181cd5d9ec83ce540 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/75725ceb5a9145b181cd5d9ec83ce540 2024-11-07T15:55:45,840 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e67c3261265748bd9df9a78a922085d9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/C/e67c3261265748bd9df9a78a922085d9 2024-11-07T15:55:45,842 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/recovered.edits/370.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b/recovered.edits/370.seqid 2024-11-07T15:55:45,842 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/8c07d911036021999ccdca9d657aee9b 2024-11-07T15:55:45,842 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:55:45,844 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,845 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:55:45,847 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:55:45,848 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,848 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:55:45,848 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994945848"}]},"ts":"9223372036854775807"} 2024-11-07T15:55:45,850 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:55:45,850 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8c07d911036021999ccdca9d657aee9b, NAME => 'TestAcidGuarantees,,1730994911412.8c07d911036021999ccdca9d657aee9b.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:55:45,850 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:55:45,850 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994945850"}]},"ts":"9223372036854775807"} 2024-11-07T15:55:45,851 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:55:45,852 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-07T15:55:45,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-07T15:55:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:55:45,934 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:45,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=237 (was 241), OpenFileDescriptor=441 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=287 (was 320), ProcessCount=11 (was 11), AvailableMemoryMB=6418 (was 5511) - AvailableMemoryMB LEAK? - 2024-11-07T15:55:45,950 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=237, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=287, ProcessCount=11, AvailableMemoryMB=6417 2024-11-07T15:55:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:55:45,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:55:45,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:45,953 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T15:55:45,953 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:45,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-07T15:55:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:45,954 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T15:55:45,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742327_1503 (size=959) 2024-11-07T15:55:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:46,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:46,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:55:46,361 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059 2024-11-07T15:55:46,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742328_1504 (size=53) 2024-11-07T15:55:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 817b58ba44d0cf51fa5fc01867ebb7a6, disabling compactions & flushes 2024-11-07T15:55:46,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. after waiting 0 ms 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:46,768 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:46,768 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 817b58ba44d0cf51fa5fc01867ebb7a6: Waiting for close lock at 1730994946768Disabling compacts and flushes for region at 1730994946768Disabling writes for close at 1730994946768Writing region close event to WAL at 1730994946768Closed at 1730994946768 2024-11-07T15:55:46,770 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T15:55:46,770 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1730994946770"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730994946770"}]},"ts":"1730994946770"} 2024-11-07T15:55:46,772 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-07T15:55:46,773 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T15:55:46,773 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994946773"}]},"ts":"1730994946773"} 2024-11-07T15:55:46,774 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-07T15:55:46,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, ASSIGN}] 2024-11-07T15:55:46,776 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, ASSIGN 2024-11-07T15:55:46,776 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, ASSIGN; state=OFFLINE, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=false 2024-11-07T15:55:46,927 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:46,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, ASSIGN because future has completed 2024-11-07T15:55:46,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:47,084 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:47,085 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:55:47,085 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,085 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:55:47,085 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,085 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,086 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,087 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName A 2024-11-07T15:55:47,087 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:47,088 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:47,088 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,089 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName B 2024-11-07T15:55:47,089 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:47,089 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:47,089 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,090 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName C 2024-11-07T15:55:47,090 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:47,090 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:47,090 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,091 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,091 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,092 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,092 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,092 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:55:47,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:47,093 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,095 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T15:55:47,095 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened 817b58ba44d0cf51fa5fc01867ebb7a6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73287643, jitterRate=0.09207098186016083}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:55:47,095 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:47,095 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for 817b58ba44d0cf51fa5fc01867ebb7a6: Running coprocessor pre-open hook at 1730994947085Writing region info on filesystem at 1730994947085Initializing all the Stores at 1730994947086 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994947086Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994947086Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994947086Cleaning up temporary data from old regions at 1730994947092 (+6 ms)Running coprocessor post-open hooks at 1730994947095 (+3 ms)Region opened successfully at 1730994947095 2024-11-07T15:55:47,096 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., pid=144, masterSystemTime=1730994947082 2024-11-07T15:55:47,098 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:47,098 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:47,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=OPEN, openSeqNum=2, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:47,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:47,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-07T15:55:47,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 in 170 msec 2024-11-07T15:55:47,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-07T15:55:47,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, ASSIGN in 326 msec 2024-11-07T15:55:47,103 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T15:55:47,103 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994947103"}]},"ts":"1730994947103"} 2024-11-07T15:55:47,104 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-07T15:55:47,105 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T15:55:47,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-11-07T15:55:48,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-07T15:55:48,103 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:48,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66320673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:48,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:48,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:48,105 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4852d424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:48,106 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:48,107 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45edf9ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:48,108 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:48,110 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:48,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T15:55:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is c2682f91c1e2,39943,1730994795853 2024-11-07T15:55:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a602226 2024-11-07T15:55:48,111 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T15:55:48,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T15:55:48,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-07T15:55:48,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T15:55:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-07T15:55:48,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742329_1505 (size=995) 2024-11-07T15:55:48,528 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-07T15:55:48,528 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-07T15:55:48,531 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:55:48,533 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE}] 2024-11-07T15:55:48,534 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE 2024-11-07T15:55:48,534 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:48,536 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE because future has completed 2024-11-07T15:55:48,536 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:55:48,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:48,688 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:48,688 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:55:48,688 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing 817b58ba44d0cf51fa5fc01867ebb7a6, disabling compactions & flushes 2024-11-07T15:55:48,688 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:48,688 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:48,689 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. after waiting 0 ms 2024-11-07T15:55:48,689 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:48,692 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T15:55:48,692 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:48,692 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for 817b58ba44d0cf51fa5fc01867ebb7a6: Waiting for close lock at 1730994948688Running coprocessor pre-close hooks at 1730994948688Disabling compacts and flushes for region at 1730994948688Disabling writes for close at 1730994948689 (+1 ms)Writing region close event to WAL at 1730994948689Running coprocessor post-close hooks at 1730994948692 (+3 ms)Closed at 1730994948692 2024-11-07T15:55:48,692 WARN [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: 817b58ba44d0cf51fa5fc01867ebb7a6 to self. 2024-11-07T15:55:48,694 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:48,694 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=CLOSED 2024-11-07T15:55:48,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:48,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-07T15:55:48,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 in 160 msec 2024-11-07T15:55:48,698 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE; state=CLOSED, location=c2682f91c1e2,42309,1730994796599; forceNewPlan=false, retain=true 2024-11-07T15:55:48,849 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=OPENING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:48,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE because future has completed 2024-11-07T15:55:48,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:55:49,005 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,005 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} 2024-11-07T15:55:49,005 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,005 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T15:55:49,005 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,005 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,006 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,007 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName A 2024-11-07T15:55:49,008 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:49,008 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:49,009 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,009 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName B 2024-11-07T15:55:49,009 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:49,010 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:49,010 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,010 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 817b58ba44d0cf51fa5fc01867ebb7a6 columnFamilyName C 2024-11-07T15:55:49,010 DEBUG [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:49,011 INFO [StoreOpener-817b58ba44d0cf51fa5fc01867ebb7a6-1 {}] regionserver.HStore(327): Store=817b58ba44d0cf51fa5fc01867ebb7a6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T15:55:49,011 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,011 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,012 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,013 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,013 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,013 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-07T15:55:49,014 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,014 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened 817b58ba44d0cf51fa5fc01867ebb7a6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63824461, jitterRate=-0.048941418528556824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-07T15:55:49,014 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,015 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for 817b58ba44d0cf51fa5fc01867ebb7a6: Running coprocessor pre-open hook at 1730994949005Writing region info on filesystem at 1730994949005Initializing all the Stores at 1730994949006 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994949006Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994949006Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730994949006Cleaning up temporary data from old regions at 1730994949013 (+7 ms)Running coprocessor post-open hooks at 1730994949014 (+1 ms)Region opened successfully at 1730994949015 (+1 ms) 2024-11-07T15:55:49,015 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., pid=149, masterSystemTime=1730994949002 2024-11-07T15:55:49,017 DEBUG [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,017 INFO [RS_OPEN_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=OPEN, openSeqNum=5, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:55:49,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-07T15:55:49,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 in 169 msec 2024-11-07T15:55:49,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-07T15:55:49,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, REOPEN/MOVE in 488 msec 2024-11-07T15:55:49,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-07T15:55:49,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-07T15:55:49,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-07T15:55:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-07T15:55:49,026 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:49,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,027 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,027 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,028 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a1b4816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,029 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c598ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,030 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,031 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@695e934d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,032 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f74e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,033 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,033 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,033 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,034 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a5ae025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,035 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e9744d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,035 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f05693, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,037 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a06a5e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f0cd5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a959704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,040 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@617b6713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,041 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,041 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,041 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15dbe4c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,042 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57290, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5530cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,043 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,043 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,044 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27fe2113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,044 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,045 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8efd4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,046 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,046 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,046 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,047 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205c302c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,048 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@352c14bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,049 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2255e167, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,051 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15eb6440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d0e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2682f91c1e2,39943,-1 for getting cluster id 2024-11-07T15:55:49,052 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T15:55:49,052 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6c0d170-2695-406e-8dc6-9b9c44c39fbc' 2024-11-07T15:55:49,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T15:55:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6c0d170-2695-406e-8dc6-9b9c44c39fbc" 2024-11-07T15:55:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f69fa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2682f91c1e2,39943,-1] 2024-11-07T15:55:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T15:55:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:55:49,053 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T15:55:49,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f199e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T15:55:49,055 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,056 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,056 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,056 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:49,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,057 DEBUG [Thread-2279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,057 DEBUG [Thread-2270 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-07T15:55:49,058 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55164, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,059 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,059 DEBUG [Thread-2270 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,060 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,060 DEBUG [Thread-2270 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,061 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55182, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,061 DEBUG [Thread-2268 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,062 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,068 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,068 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T15:55:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2682f91c1e2,42309,1730994796599, seqNum=-1] 2024-11-07T15:55:49,070 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T15:55:49,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T15:55:49,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5] 2024-11-07T15:55:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,073 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:49,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-07T15:55:49,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bcbf9d4edafa49b5b6cdafcdfa4375b7_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994949066/Put/seqid=0 2024-11-07T15:55:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995009103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,107 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995009103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,108 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995009105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995009106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,111 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995009107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742330_1506 (size=12154) 2024-11-07T15:55:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:49,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995009213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,214 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995009213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995009214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995009214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995009222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995009423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995009423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995009424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995009424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995009443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,514 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:49,518 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bcbf9d4edafa49b5b6cdafcdfa4375b7_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bcbf9d4edafa49b5b6cdafcdfa4375b7_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:49,519 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/54a4c800c01740fa9a201f497fa0181d, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:49,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/54a4c800c01740fa9a201f497fa0181d is 175, key is test_row_0/A:col10/1730994949066/Put/seqid=0 2024-11-07T15:55:49,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742331_1507 (size=30955) 2024-11-07T15:55:49,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/54a4c800c01740fa9a201f497fa0181d 2024-11-07T15:55:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2f455ad389604466813de377d175815e is 50, key is test_row_0/B:col10/1730994949066/Put/seqid=0 2024-11-07T15:55:49,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742332_1508 (size=12001) 2024-11-07T15:55:49,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:49,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995009733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995009733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995009733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995009733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995009753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:49,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2f455ad389604466813de377d175815e 2024-11-07T15:55:49,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/8e2ac3d276344b45b8ae7763780bcb21 is 50, key is test_row_0/C:col10/1730994949066/Put/seqid=0 2024-11-07T15:55:49,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742333_1509 (size=12001) 2024-11-07T15:55:49,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:49,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:49,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:49,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:50,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:50,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995010242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:50,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995010243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995010243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:50,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995010243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:50,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995010263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:50,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:50,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:50,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/8e2ac3d276344b45b8ae7763780bcb21 2024-11-07T15:55:50,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/54a4c800c01740fa9a201f497fa0181d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d 2024-11-07T15:55:50,386 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d, entries=150, sequenceid=16, filesize=30.2 K 2024-11-07T15:55:50,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2f455ad389604466813de377d175815e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e 2024-11-07T15:55:50,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:55:50,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/8e2ac3d276344b45b8ae7763780bcb21 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21 2024-11-07T15:55:50,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21, entries=150, sequenceid=16, filesize=11.7 K 2024-11-07T15:55:50,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1321ms, sequenceid=16, compaction requested=false 2024-11-07T15:55:50,394 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-07T15:55:50,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:50,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-07T15:55:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:50,442 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-07T15:55:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076175338460f240f4866eeff9399b12c4_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994949105/Put/seqid=0 2024-11-07T15:55:50,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742334_1510 (size=12154) 2024-11-07T15:55:50,687 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T15:55:50,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:50,857 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076175338460f240f4866eeff9399b12c4_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076175338460f240f4866eeff9399b12c4_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:50,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/3b8caaefd1ab4843bbe4bf4fc6a2b713, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:50,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 is 175, key is test_row_0/A:col10/1730994949105/Put/seqid=0 2024-11-07T15:55:50,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742335_1511 (size=30955) 2024-11-07T15:55:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:51,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:51,262 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 2024-11-07T15:55:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995011263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/fa13f63238b346fda2aa47c84c5812ff is 50, key is test_row_0/B:col10/1730994949105/Put/seqid=0 2024-11-07T15:55:51,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995011267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,271 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995011268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995011269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995011273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742336_1512 (size=12001) 2024-11-07T15:55:51,275 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995011373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995011373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995011383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995011383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995011583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,584 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995011583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995011593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995011593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,675 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/fa13f63238b346fda2aa47c84c5812ff 2024-11-07T15:55:51,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/d6b46425b08a47baa322fc4f5f9a1af3 is 50, key is test_row_0/C:col10/1730994949105/Put/seqid=0 2024-11-07T15:55:51,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742337_1513 (size=12001) 2024-11-07T15:55:51,688 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/d6b46425b08a47baa322fc4f5f9a1af3 2024-11-07T15:55:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 2024-11-07T15:55:51,695 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713, entries=150, sequenceid=40, filesize=30.2 K 2024-11-07T15:55:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/fa13f63238b346fda2aa47c84c5812ff as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff 2024-11-07T15:55:51,700 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:55:51,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/d6b46425b08a47baa322fc4f5f9a1af3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3 2024-11-07T15:55:51,703 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3, entries=150, sequenceid=40, filesize=11.7 K 2024-11-07T15:55:51,704 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1262ms, sequenceid=40, compaction requested=false 2024-11-07T15:55:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-07T15:55:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-07T15:55:51,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-07T15:55:51,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6460 sec 2024-11-07T15:55:51,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.6520 sec 2024-11-07T15:55:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:51,895 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:51,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:55:51,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075b3f8723707e4e2cbf751b1d3892d1e9_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:51,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742338_1514 (size=14594) 2024-11-07T15:55:51,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995011920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995011921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,925 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995011923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,927 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:51,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995011924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:51,929 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995012033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995012033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995012033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,037 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995012033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995012243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995012243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995012243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995012243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:52,315 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411075b3f8723707e4e2cbf751b1d3892d1e9_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075b3f8723707e4e2cbf751b1d3892d1e9_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:52,316 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2ddb93b89d6d4f5e8e00bdf3f42e1334, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:52,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 is 175, key is test_row_0/A:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:52,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742339_1515 (size=39549) 2024-11-07T15:55:52,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995012552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995012553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995012553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995012553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:52,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:52,721 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 2024-11-07T15:55:52,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/00f8483839b64e179ad8531f6f508939 is 50, key is test_row_0/B:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:52,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742340_1516 (size=12001) 2024-11-07T15:55:52,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/00f8483839b64e179ad8531f6f508939 2024-11-07T15:55:52,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4cb591ad3f6c435db67ee4380ef6582d is 50, key is test_row_0/C:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:52,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742341_1517 (size=12001) 2024-11-07T15:55:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995013062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995013063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995013063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995013063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4cb591ad3f6c435db67ee4380ef6582d 2024-11-07T15:55:53,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 2024-11-07T15:55:53,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334, entries=200, sequenceid=55, filesize=38.6 K 2024-11-07T15:55:53,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/00f8483839b64e179ad8531f6f508939 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939 2024-11-07T15:55:53,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:55:53,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4cb591ad3f6c435db67ee4380ef6582d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d 2024-11-07T15:55:53,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d, entries=150, sequenceid=55, filesize=11.7 K 2024-11-07T15:55:53,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1266ms, sequenceid=55, compaction requested=true 2024-11-07T15:55:53,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:53,162 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:53,162 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:53,162 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:53,162 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/A is initiating minor compaction (all files) 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/B is initiating minor compaction (all files) 2024-11-07T15:55:53,163 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/A in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,163 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/B in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,163 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=99.1 K 2024-11-07T15:55:53,163 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=35.2 K 2024-11-07T15:55:53,163 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334] 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f455ad389604466813de377d175815e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994949066 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54a4c800c01740fa9a201f497fa0181d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994949066 2024-11-07T15:55:53,163 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fa13f63238b346fda2aa47c84c5812ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994949098 2024-11-07T15:55:53,164 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8caaefd1ab4843bbe4bf4fc6a2b713, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994949098 2024-11-07T15:55:53,164 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 00f8483839b64e179ad8531f6f508939, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:53,164 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ddb93b89d6d4f5e8e00bdf3f42e1334, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:53,170 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:53,171 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1baf5c37b0e64d3d9d70099c28facff0 is 50, key is test_row_0/B:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:53,171 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:53,174 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107e7e49c94900c4a05b5f75bc6590f9733_817b58ba44d0cf51fa5fc01867ebb7a6 store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:53,176 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107e7e49c94900c4a05b5f75bc6590f9733_817b58ba44d0cf51fa5fc01867ebb7a6, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:53,177 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107e7e49c94900c4a05b5f75bc6590f9733_817b58ba44d0cf51fa5fc01867ebb7a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:53,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742342_1518 (size=12104) 2024-11-07T15:55:53,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742343_1519 (size=4469) 2024-11-07T15:55:53,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-07T15:55:53,223 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:53,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-07T15:55:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:53,226 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:53,227 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:53,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:53,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:53,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-07T15:55:53,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107139b79a941ca42888889ee142805a0b2_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994953304/Put/seqid=0 2024-11-07T15:55:53,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742344_1520 (size=12154) 2024-11-07T15:55:53,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:53,320 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107139b79a941ca42888889ee142805a0b2_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107139b79a941ca42888889ee142805a0b2_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:53,320 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/90e241358cc14ac8a9c7af1785f1b336, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:53,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/90e241358cc14ac8a9c7af1785f1b336 is 175, key is test_row_0/A:col10/1730994953304/Put/seqid=0 2024-11-07T15:55:53,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742345_1521 (size=30955) 2024-11-07T15:55:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:53,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995013361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995013473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:53,594 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1baf5c37b0e64d3d9d70099c28facff0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1baf5c37b0e64d3d9d70099c28facff0 2024-11-07T15:55:53,598 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/B of 817b58ba44d0cf51fa5fc01867ebb7a6 into 1baf5c37b0e64d3d9d70099c28facff0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:53,598 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:53,598 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/B, priority=13, startTime=1730994953162; duration=0sec 2024-11-07T15:55:53,598 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:53,598 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:55:53,598 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:55:53,599 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:55:53,599 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/C is initiating minor compaction (all files) 2024-11-07T15:55:53,599 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/C in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,599 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=35.2 K 2024-11-07T15:55:53,600 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e2ac3d276344b45b8ae7763780bcb21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1730994949066 2024-11-07T15:55:53,600 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b46425b08a47baa322fc4f5f9a1af3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1730994949098 2024-11-07T15:55:53,600 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cb591ad3f6c435db67ee4380ef6582d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:53,607 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:53,607 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#C#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:53,607 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/61c20ad198d64da39ecd756711f68a2a is 175, key is test_row_0/A:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:53,607 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/23d62d7cdb574495836d226cc615c4d5 is 50, key is test_row_0/C:col10/1730994951256/Put/seqid=0 2024-11-07T15:55:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742346_1522 (size=31058) 2024-11-07T15:55:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742347_1523 (size=12104) 2024-11-07T15:55:53,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:53,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995013683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,725 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/90e241358cc14ac8a9c7af1785f1b336 2024-11-07T15:55:53,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bbfd82bbe2304bfb806546429ca9efd8 is 50, key is test_row_0/B:col10/1730994953304/Put/seqid=0 2024-11-07T15:55:53,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742348_1524 (size=12001) 2024-11-07T15:55:53,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:53,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:53,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:53,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995013992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,024 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/61c20ad198d64da39ecd756711f68a2a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a 2024-11-07T15:55:54,024 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/23d62d7cdb574495836d226cc615c4d5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/23d62d7cdb574495836d226cc615c4d5 2024-11-07T15:55:54,028 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/C of 817b58ba44d0cf51fa5fc01867ebb7a6 into 23d62d7cdb574495836d226cc615c4d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:54,028 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/A of 817b58ba44d0cf51fa5fc01867ebb7a6 into 61c20ad198d64da39ecd756711f68a2a(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:54,028 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/A, priority=13, startTime=1730994953162; duration=0sec 2024-11-07T15:55:54,028 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/C, priority=13, startTime=1730994953162; duration=0sec 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:54,028 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:55:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995014073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995014073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995014073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,077 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,078 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:54,078 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995014082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,085 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:54,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,085 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bbfd82bbe2304bfb806546429ca9efd8 2024-11-07T15:55:54,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/3740b0b8319a417f8f2748fdb5c94130 is 50, key is test_row_0/C:col10/1730994953304/Put/seqid=0 2024-11-07T15:55:54,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:54,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:54,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:55:54,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742349_1525 (size=12001) 2024-11-07T15:55:54,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/3740b0b8319a417f8f2748fdb5c94130 2024-11-07T15:55:54,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/90e241358cc14ac8a9c7af1785f1b336 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336 2024-11-07T15:55:54,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336, entries=150, sequenceid=77, filesize=30.2 K 2024-11-07T15:55:54,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bbfd82bbe2304bfb806546429ca9efd8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8 2024-11-07T15:55:54,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:55:54,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/3740b0b8319a417f8f2748fdb5c94130 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130 2024-11-07T15:55:54,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130, entries=150, sequenceid=77, filesize=11.7 K 2024-11-07T15:55:54,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 858ms, sequenceid=77, compaction requested=false 2024-11-07T15:55:54,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:54,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-07T15:55:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:54,298 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-07T15:55:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107474cca62c59d45b6afd6cae37ac55937_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994953360/Put/seqid=0 2024-11-07T15:55:54,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742350_1526 (size=12154) 2024-11-07T15:55:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:54,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:54,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995014593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995014703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:54,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:54,719 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107474cca62c59d45b6afd6cae37ac55937_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107474cca62c59d45b6afd6cae37ac55937_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:54,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/fa4f19fff0584c349e336713ebba69e2, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/fa4f19fff0584c349e336713ebba69e2 is 175, key is test_row_0/A:col10/1730994953360/Put/seqid=0 2024-11-07T15:55:54,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742351_1527 (size=30955) 2024-11-07T15:55:54,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995014913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:54,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:55,124 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/fa4f19fff0584c349e336713ebba69e2 2024-11-07T15:55:55,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/476caa65a9714f688964a6b148234fc0 is 50, key is test_row_0/B:col10/1730994953360/Put/seqid=0 2024-11-07T15:55:55,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742352_1528 (size=12001) 2024-11-07T15:55:55,135 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/476caa65a9714f688964a6b148234fc0 2024-11-07T15:55:55,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/b36f29e2bce84baaa65e1ec9ca729bae is 50, key is test_row_0/C:col10/1730994953360/Put/seqid=0 2024-11-07T15:55:55,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742353_1529 (size=12001) 2024-11-07T15:55:55,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995015223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:55,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:55,544 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/b36f29e2bce84baaa65e1ec9ca729bae 2024-11-07T15:55:55,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/fa4f19fff0584c349e336713ebba69e2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2 2024-11-07T15:55:55,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2, entries=150, sequenceid=94, filesize=30.2 K 2024-11-07T15:55:55,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/476caa65a9714f688964a6b148234fc0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0 2024-11-07T15:55:55,556 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:55:55,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/b36f29e2bce84baaa65e1ec9ca729bae as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae 2024-11-07T15:55:55,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae, entries=150, sequenceid=94, filesize=11.7 K 2024-11-07T15:55:55,560 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1262ms, sequenceid=94, compaction requested=true 2024-11-07T15:55:55,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:55,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:55,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-07T15:55:55,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-07T15:55:55,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-07T15:55:55,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3350 sec 2024-11-07T15:55:55,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.3400 sec 2024-11-07T15:55:55,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:55,737 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:55:55,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:55:55,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079771f684fa9b4024ac7bc8a2b7ff9cc9_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:55,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742354_1530 (size=14594) 2024-11-07T15:55:55,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:55,751 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411079771f684fa9b4024ac7bc8a2b7ff9cc9_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079771f684fa9b4024ac7bc8a2b7ff9cc9_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:55,751 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b126ad3836f6403392cdb51d786a5255, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:55,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b126ad3836f6403392cdb51d786a5255 is 175, key is test_row_0/A:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:55,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742355_1531 (size=39549) 2024-11-07T15:55:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995015782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:55,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995015893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995016093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995016093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,097 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_2' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,097 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_0' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,097 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995016103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995016103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995016103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,106 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:56,106 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,156 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b126ad3836f6403392cdb51d786a5255 2024-11-07T15:55:56,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bef2ad0ff2434865b9a1931616d161b8 is 50, key is test_row_0/B:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:56,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742356_1532 (size=12001) 2024-11-07T15:55:56,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:55:56,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-07T15:55:56,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995016412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bef2ad0ff2434865b9a1931616d161b8 2024-11-07T15:55:56,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/50ad609a0f644151bc64eae7b9adfef5 is 50, key is test_row_0/C:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:56,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742357_1533 (size=12001) 2024-11-07T15:55:56,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995016923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:56,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:56,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/50ad609a0f644151bc64eae7b9adfef5 2024-11-07T15:55:56,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b126ad3836f6403392cdb51d786a5255 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255 2024-11-07T15:55:56,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255, entries=200, sequenceid=117, filesize=38.6 K 2024-11-07T15:55:56,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/bef2ad0ff2434865b9a1931616d161b8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8 2024-11-07T15:55:56,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T15:55:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/50ad609a0f644151bc64eae7b9adfef5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5 2024-11-07T15:55:56,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-07T15:55:56,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1259ms, sequenceid=117, compaction requested=true 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:55:56,996 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:55:56,996 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:56,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:56,997 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:56,997 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:56,997 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/A is initiating minor compaction (all files) 2024-11-07T15:55:56,997 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/B is initiating minor compaction (all files) 2024-11-07T15:55:56,997 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/B in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:56,997 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/A in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:56,997 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1baf5c37b0e64d3d9d70099c28facff0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=47.0 K 2024-11-07T15:55:56,997 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=129.4 K 2024-11-07T15:55:56,998 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:56,998 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255] 2024-11-07T15:55:56,998 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1baf5c37b0e64d3d9d70099c28facff0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:56,998 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61c20ad198d64da39ecd756711f68a2a, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:56,998 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting bbfd82bbe2304bfb806546429ca9efd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730994951922 2024-11-07T15:55:56,998 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e241358cc14ac8a9c7af1785f1b336, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730994951922 2024-11-07T15:55:56,999 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 476caa65a9714f688964a6b148234fc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730994953318 2024-11-07T15:55:56,999 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa4f19fff0584c349e336713ebba69e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730994953318 2024-11-07T15:55:56,999 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b126ad3836f6403392cdb51d786a5255, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:55:56,999 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting bef2ad0ff2434865b9a1931616d161b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:55:57,004 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:57,006 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107d67967aaa2154b9184c143300f293106_817b58ba44d0cf51fa5fc01867ebb7a6 store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:57,006 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:57,007 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ca9a8b2e2a314772885ad548cad592e5 is 50, key is test_row_0/B:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:57,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107d67967aaa2154b9184c143300f293106_817b58ba44d0cf51fa5fc01867ebb7a6, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:57,009 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107d67967aaa2154b9184c143300f293106_817b58ba44d0cf51fa5fc01867ebb7a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:57,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742358_1534 (size=12241) 2024-11-07T15:55:57,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742359_1535 (size=4469) 2024-11-07T15:55:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-07T15:55:57,383 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:57,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-07T15:55:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:57,385 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:57,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:57,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:57,414 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#A#compaction#447 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:57,414 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/cb51189eba9d4e9f8ff6fe039a3eedad is 175, key is test_row_0/A:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:57,415 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ca9a8b2e2a314772885ad548cad592e5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ca9a8b2e2a314772885ad548cad592e5 2024-11-07T15:55:57,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742360_1536 (size=31195) 2024-11-07T15:55:57,420 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/B of 817b58ba44d0cf51fa5fc01867ebb7a6 into ca9a8b2e2a314772885ad548cad592e5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:57,420 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:57,420 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/B, priority=12, startTime=1730994956996; duration=0sec 2024-11-07T15:55:57,420 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:55:57,420 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:55:57,420 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:55:57,421 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:55:57,421 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/C is initiating minor compaction (all files) 2024-11-07T15:55:57,422 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/C in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:57,422 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/23d62d7cdb574495836d226cc615c4d5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=47.0 K 2024-11-07T15:55:57,422 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d62d7cdb574495836d226cc615c4d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1730994951256 2024-11-07T15:55:57,422 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 3740b0b8319a417f8f2748fdb5c94130, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1730994951922 2024-11-07T15:55:57,423 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/cb51189eba9d4e9f8ff6fe039a3eedad as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad 2024-11-07T15:55:57,423 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting b36f29e2bce84baaa65e1ec9ca729bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1730994953318 2024-11-07T15:55:57,423 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 50ad609a0f644151bc64eae7b9adfef5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:55:57,426 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/A of 817b58ba44d0cf51fa5fc01867ebb7a6 into cb51189eba9d4e9f8ff6fe039a3eedad(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:57,426 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:57,427 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/A, priority=12, startTime=1730994956996; duration=0sec 2024-11-07T15:55:57,427 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:57,427 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:55:57,430 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:55:57,431 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/2c859c9fea2d41f29ec9484d8276b41a is 50, key is test_row_0/C:col10/1730994954591/Put/seqid=0 2024-11-07T15:55:57,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742361_1537 (size=12241) 2024-11-07T15:55:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:57,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-07T15:55:57,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:57,538 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-07T15:55:57,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f13d4281cf384892bc40d2ac015d3512_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994955781/Put/seqid=0 2024-11-07T15:55:57,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742362_1538 (size=12204) 2024-11-07T15:55:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:57,843 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/2c859c9fea2d41f29ec9484d8276b41a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2c859c9fea2d41f29ec9484d8276b41a 2024-11-07T15:55:57,847 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/C of 817b58ba44d0cf51fa5fc01867ebb7a6 into 2c859c9fea2d41f29ec9484d8276b41a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:55:57,847 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:57,847 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/C, priority=12, startTime=1730994956996; duration=0sec 2024-11-07T15:55:57,847 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:55:57,847 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:55:57,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:55:57,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:57,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:55:57,952 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f13d4281cf384892bc40d2ac015d3512_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f13d4281cf384892bc40d2ac015d3512_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:55:57,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b35b289fea79446385634bc1c680dfae, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:55:57,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b35b289fea79446385634bc1c680dfae is 175, key is test_row_0/A:col10/1730994955781/Put/seqid=0 2024-11-07T15:55:57,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742363_1539 (size=31005) 2024-11-07T15:55:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:58,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995018016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:58,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995018123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:58,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995018333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:58,357 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b35b289fea79446385634bc1c680dfae 2024-11-07T15:55:58,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1d7a807b420e49d98eb5906d38da91a7 is 50, key is test_row_0/B:col10/1730994955781/Put/seqid=0 2024-11-07T15:55:58,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742364_1540 (size=12051) 2024-11-07T15:55:58,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:58,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:58,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995018642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:58,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:58,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:58,768 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1d7a807b420e49d98eb5906d38da91a7 2024-11-07T15:55:58,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/ea6ad55f6e0a4548ad59bab18872205b is 50, key is test_row_0/C:col10/1730994955781/Put/seqid=0 2024-11-07T15:55:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742365_1541 (size=12051) 2024-11-07T15:55:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:55:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995019153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:55:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:55:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:55:59,178 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/ea6ad55f6e0a4548ad59bab18872205b 2024-11-07T15:55:59,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/b35b289fea79446385634bc1c680dfae as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae 2024-11-07T15:55:59,185 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae, entries=150, sequenceid=132, filesize=30.3 K 2024-11-07T15:55:59,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1d7a807b420e49d98eb5906d38da91a7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7 2024-11-07T15:55:59,189 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T15:55:59,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/ea6ad55f6e0a4548ad59bab18872205b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b 2024-11-07T15:55:59,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b, entries=150, sequenceid=132, filesize=11.8 K 2024-11-07T15:55:59,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1655ms, sequenceid=132, compaction requested=false 2024-11-07T15:55:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:55:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:59,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-07T15:55:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-07T15:55:59,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-07T15:55:59,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-07T15:55:59,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.8140 sec 2024-11-07T15:55:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-07T15:55:59,533 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:55:59,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:55:59,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-07T15:55:59,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:55:59,535 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:55:59,536 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:55:59,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:55:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:55:59,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-07T15:55:59,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:55:59,688 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-07T15:55:59,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d83e992ce9f4d9c8060a0e04724c138_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994958001/Put/seqid=0 2024-11-07T15:55:59,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742366_1542 (size=12304) 2024-11-07T15:55:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:56:00,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:00,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411074d83e992ce9f4d9c8060a0e04724c138_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83e992ce9f4d9c8060a0e04724c138_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:00,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/978ea5d3e74e45e88bd33ae9ec1879c1, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:00,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/978ea5d3e74e45e88bd33ae9ec1879c1 is 175, key is test_row_0/A:col10/1730994958001/Put/seqid=0 2024-11-07T15:56:00,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:00,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:00,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742367_1543 (size=31105) 2024-11-07T15:56:00,107 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/978ea5d3e74e45e88bd33ae9ec1879c1 2024-11-07T15:56:00,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2c0741412ee3490d94e21b8d8d6df4e8 is 50, key is test_row_0/B:col10/1730994958001/Put/seqid=0 2024-11-07T15:56:00,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742368_1544 (size=12151) 2024-11-07T15:56:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995020127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995020128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995020128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55220 deadline: 1730995020133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,136 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:56:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:56:00,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995020163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,166 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:56:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995020243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995020243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995020243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995020453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995020453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995020453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,518 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2c0741412ee3490d94e21b8d8d6df4e8 2024-11-07T15:56:00,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/03ccd6f83b4a4d75aef075020fff863c is 50, key is test_row_0/C:col10/1730994958001/Put/seqid=0 2024-11-07T15:56:00,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742369_1545 (size=12151) 2024-11-07T15:56:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:56:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995020763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995020763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995020763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:00,928 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/03ccd6f83b4a4d75aef075020fff863c 2024-11-07T15:56:00,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/978ea5d3e74e45e88bd33ae9ec1879c1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1 2024-11-07T15:56:00,936 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1, entries=150, sequenceid=156, filesize=30.4 K 2024-11-07T15:56:00,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/2c0741412ee3490d94e21b8d8d6df4e8 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8 2024-11-07T15:56:00,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T15:56:00,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/03ccd6f83b4a4d75aef075020fff863c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c 2024-11-07T15:56:00,943 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-07T15:56:00,944 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1256ms, sequenceid=156, compaction requested=true 2024-11-07T15:56:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:00,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-07T15:56:00,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-07T15:56:00,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-07T15:56:00,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-07T15:56:00,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4130 sec 2024-11-07T15:56:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:01,276 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:56:01,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-07T15:56:01,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110767244022c8be4b558c94a2db38d01e94_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:01,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742370_1546 (size=12304) 2024-11-07T15:56:01,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995021306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995021306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,311 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995021308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,312 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995021413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995021413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995021423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995021623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995021623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995021633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-07T15:56:01,683 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:01,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:56:01,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-07T15:56:01,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:01,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:56:01,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:56:01,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:56:01,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:01,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110767244022c8be4b558c94a2db38d01e94_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110767244022c8be4b558c94a2db38d01e94_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:01,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/4e538ac39dac47f19c24c1bcb7ce6a9b, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:01,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/4e538ac39dac47f19c24c1bcb7ce6a9b is 175, key is test_row_0/A:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:01,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742371_1547 (size=31105) 2024-11-07T15:56:01,695 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/4e538ac39dac47f19c24c1bcb7ce6a9b 2024-11-07T15:56:01,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/74a5b3d1752647a19cb85c86bf9feff0 is 50, key is test_row_0/B:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:01,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742372_1548 (size=12151) 2024-11-07T15:56:01,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:01,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:01,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:01,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:01,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995021933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995021933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:01,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995021943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:01,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:01,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:01,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:01,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:02,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/74a5b3d1752647a19cb85c86bf9feff0 2024-11-07T15:56:02,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/dfd4604b0a6145489132e24164a421c0 is 50, key is test_row_0/C:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:02,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742373_1549 (size=12151) 2024-11-07T15:56:02,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995022183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:02,187 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to c2682f91c1e2,42309,1730994796599 for 'test_row_1' in 817b58ba44d0cf51fa5fc01867ebb7a6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-07T15:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:02,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:02,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995022443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995022443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:02,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995022453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:02,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/dfd4604b0a6145489132e24164a421c0 2024-11-07T15:56:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/4e538ac39dac47f19c24c1bcb7ce6a9b as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b 2024-11-07T15:56:02,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b, entries=150, sequenceid=172, filesize=30.4 K 2024-11-07T15:56:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/74a5b3d1752647a19cb85c86bf9feff0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0 2024-11-07T15:56:02,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T15:56:02,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/dfd4604b0a6145489132e24164a421c0 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0 2024-11-07T15:56:02,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-07T15:56:02,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1261ms, sequenceid=172, compaction requested=true 2024-11-07T15:56:02,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:56:02,538 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:56:02,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:02,538 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/B is initiating minor compaction (all files) 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/A is initiating minor compaction (all files) 2024-11-07T15:56:02,539 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/B in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,539 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/A in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,539 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ca9a8b2e2a314772885ad548cad592e5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=47.5 K 2024-11-07T15:56:02,539 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=121.5 K 2024-11-07T15:56:02,539 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b] 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9a8b2e2a314772885ad548cad592e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:56:02,539 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb51189eba9d4e9f8ff6fe039a3eedad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d7a807b420e49d98eb5906d38da91a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994955765 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting b35b289fea79446385634bc1c680dfae, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994955765 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c0741412ee3490d94e21b8d8d6df4e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730994957987 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 978ea5d3e74e45e88bd33ae9ec1879c1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730994957987 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a5b3d1752647a19cb85c86bf9feff0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:02,540 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e538ac39dac47f19c24c1bcb7ce6a9b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:02,551 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:02,553 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:02,553 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/56a0fd8e01df43d7aabd77379c1a091f is 50, key is test_row_0/B:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:02,554 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bc0f45240a244820a94c671676082cbe_817b58ba44d0cf51fa5fc01867ebb7a6 store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:02,556 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bc0f45240a244820a94c671676082cbe_817b58ba44d0cf51fa5fc01867ebb7a6, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:02,556 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bc0f45240a244820a94c671676082cbe_817b58ba44d0cf51fa5fc01867ebb7a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:02,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742374_1550 (size=12527) 2024-11-07T15:56:02,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742375_1551 (size=4469) 2024-11-07T15:56:02,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-07T15:56:02,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,603 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:56:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a7c0dd5f000469b9fba6639d47e79fd_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994961299/Put/seqid=0 2024-11-07T15:56:02,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742376_1552 (size=12304) 2024-11-07T15:56:02,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:02,968 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/56a0fd8e01df43d7aabd77379c1a091f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/56a0fd8e01df43d7aabd77379c1a091f 2024-11-07T15:56:02,972 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/B of 817b58ba44d0cf51fa5fc01867ebb7a6 into 56a0fd8e01df43d7aabd77379c1a091f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:02,972 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:02,972 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/B, priority=12, startTime=1730994962538; duration=0sec 2024-11-07T15:56:02,972 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:02,972 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:56:02,973 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-07T15:56:02,973 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48594 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-07T15:56:02,974 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/C is initiating minor compaction (all files) 2024-11-07T15:56:02,974 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/C in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:02,974 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2c859c9fea2d41f29ec9484d8276b41a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=47.5 K 2024-11-07T15:56:02,974 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c859c9fea2d41f29ec9484d8276b41a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1730994954580 2024-11-07T15:56:02,974 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6ad55f6e0a4548ad59bab18872205b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1730994955765 2024-11-07T15:56:02,975 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#A#compaction#460 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:02,975 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 03ccd6f83b4a4d75aef075020fff863c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1730994957987 2024-11-07T15:56:02,975 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd4604b0a6145489132e24164a421c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:02,975 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bff1a65691214afeb1db021f8c40a44f is 175, key is test_row_0/A:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:02,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742377_1553 (size=31481) 2024-11-07T15:56:02,984 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#C#compaction#462 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:02,985 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/566d91c99b624e91ae3c72d764642ab2 is 50, key is test_row_0/C:col10/1730994961275/Put/seqid=0 2024-11-07T15:56:02,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742378_1554 (size=12527) 2024-11-07T15:56:02,991 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/566d91c99b624e91ae3c72d764642ab2 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/566d91c99b624e91ae3c72d764642ab2 2024-11-07T15:56:02,994 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/C of 817b58ba44d0cf51fa5fc01867ebb7a6 into 566d91c99b624e91ae3c72d764642ab2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:02,994 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:02,994 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/C, priority=12, startTime=1730994962538; duration=0sec 2024-11-07T15:56:02,994 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:02,994 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:56:03,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:03,016 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411076a7c0dd5f000469b9fba6639d47e79fd_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a7c0dd5f000469b9fba6639d47e79fd_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:03,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2cc94be729d842fab4594c0588131e9e, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:03,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2cc94be729d842fab4594c0588131e9e is 175, key is test_row_0/A:col10/1730994961299/Put/seqid=0 2024-11-07T15:56:03,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742379_1555 (size=31105) 2024-11-07T15:56:03,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2cc94be729d842fab4594c0588131e9e 2024-11-07T15:56:03,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ec9e24ceff4f4f25ad127fa716851837 is 50, key is test_row_0/B:col10/1730994961299/Put/seqid=0 2024-11-07T15:56:03,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742380_1556 (size=12151) 2024-11-07T15:56:03,390 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bff1a65691214afeb1db021f8c40a44f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f 2024-11-07T15:56:03,394 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/A of 817b58ba44d0cf51fa5fc01867ebb7a6 into bff1a65691214afeb1db021f8c40a44f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:03,394 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:03,394 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/A, priority=12, startTime=1730994962537; duration=0sec 2024-11-07T15:56:03,395 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:03,395 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:56:03,431 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ec9e24ceff4f4f25ad127fa716851837 2024-11-07T15:56:03,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9fd824f7d58e48a5a7f00b262c7440d4 is 50, key is test_row_0/C:col10/1730994961299/Put/seqid=0 2024-11-07T15:56:03,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742381_1557 (size=12151) 2024-11-07T15:56:03,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:03,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995023476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995023478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995023479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995023593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995023593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995023593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995023803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995023803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995023803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:03,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:03,842 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9fd824f7d58e48a5a7f00b262c7440d4 2024-11-07T15:56:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/2cc94be729d842fab4594c0588131e9e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e 2024-11-07T15:56:03,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e, entries=150, sequenceid=192, filesize=30.4 K 2024-11-07T15:56:03,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ec9e24ceff4f4f25ad127fa716851837 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837 2024-11-07T15:56:03,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T15:56:03,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9fd824f7d58e48a5a7f00b262c7440d4 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4 2024-11-07T15:56:03,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4, entries=150, sequenceid=192, filesize=11.9 K 2024-11-07T15:56:03,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1262ms, sequenceid=192, compaction requested=false 2024-11-07T15:56:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-07T15:56:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-07T15:56:03,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-07T15:56:03,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1810 sec 2024-11-07T15:56:03,870 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.1860 sec 2024-11-07T15:56:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:04,115 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:56:04,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-07T15:56:04,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cfaa71f780a04bc4b98043f6f903a5ca_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:04,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742382_1558 (size=17284) 2024-11-07T15:56:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995024138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995024139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995024142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995024253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995024253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995024253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995024463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995024463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995024463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:04,531 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107cfaa71f780a04bc4b98043f6f903a5ca_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfaa71f780a04bc4b98043f6f903a5ca_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:04,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/e2344d5595764fb18cba3e7f12a396a1, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:04,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/e2344d5595764fb18cba3e7f12a396a1 is 175, key is test_row_0/A:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:04,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742383_1559 (size=48389) 2024-11-07T15:56:04,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995024773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995024773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995024773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:04,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:04,937 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/e2344d5595764fb18cba3e7f12a396a1 2024-11-07T15:56:04,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ef2163df93334566822011050fcea0b5 is 50, key is test_row_0/B:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:04,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742384_1560 (size=12151) 2024-11-07T15:56:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995025283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995025283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:05,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:05,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995025283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:05,286 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:05,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ef2163df93334566822011050fcea0b5 2024-11-07T15:56:05,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/5fb13ba6947242049d423dadb91a394f is 50, key is test_row_0/C:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:05,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742385_1561 (size=12151) 2024-11-07T15:56:05,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/5fb13ba6947242049d423dadb91a394f 2024-11-07T15:56:05,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/e2344d5595764fb18cba3e7f12a396a1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1 2024-11-07T15:56:05,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1, entries=250, sequenceid=213, filesize=47.3 K 2024-11-07T15:56:05,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/ef2163df93334566822011050fcea0b5 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5 2024-11-07T15:56:05,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T15:56:05,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/5fb13ba6947242049d423dadb91a394f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f 2024-11-07T15:56:05,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f, entries=150, sequenceid=213, filesize=11.9 K 2024-11-07T15:56:05,776 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1661ms, sequenceid=213, compaction requested=true 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:05,776 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:56:05,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:05,776 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:05,777 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:05,777 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:05,777 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/B is initiating minor compaction (all files) 2024-11-07T15:56:05,777 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/A is initiating minor compaction (all files) 2024-11-07T15:56:05,777 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/A in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:05,777 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/B in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:05,777 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=108.4 K 2024-11-07T15:56:05,777 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/56a0fd8e01df43d7aabd77379c1a091f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=36.0 K 2024-11-07T15:56:05,777 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:05,778 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1] 2024-11-07T15:56:05,778 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a0fd8e01df43d7aabd77379c1a091f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:05,778 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting bff1a65691214afeb1db021f8c40a44f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:05,778 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ec9e24ceff4f4f25ad127fa716851837, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730994961299 2024-11-07T15:56:05,778 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cc94be729d842fab4594c0588131e9e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730994961299 2024-11-07T15:56:05,779 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2344d5595764fb18cba3e7f12a396a1, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:05,779 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2163df93334566822011050fcea0b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:05,786 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:05,786 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:05,786 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/980315b15ccb4524bb7d97c68537275c is 50, key is test_row_0/B:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:05,787 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411070fdd94a5c5b04bd78ff0e54aaacbff31_817b58ba44d0cf51fa5fc01867ebb7a6 store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:05,789 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411070fdd94a5c5b04bd78ff0e54aaacbff31_817b58ba44d0cf51fa5fc01867ebb7a6, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:05,789 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411070fdd94a5c5b04bd78ff0e54aaacbff31_817b58ba44d0cf51fa5fc01867ebb7a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:05,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742386_1562 (size=12629) 2024-11-07T15:56:05,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742387_1563 (size=4469) 2024-11-07T15:56:05,795 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#A#compaction#469 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:05,795 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/db78caa4e5644342b2d7d88301d3b699 is 175, key is test_row_0/A:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:05,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742388_1564 (size=31583) 2024-11-07T15:56:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-07T15:56:05,863 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:05,864 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:56:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-07T15:56:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:05,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:56:05,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:56:05,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:56:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:06,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-07T15:56:06,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:06,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:56:06,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110738728dfba0eb4c2ab783b56fae6b6a42_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994964137/Put/seqid=0 2024-11-07T15:56:06,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742389_1565 (size=12304) 2024-11-07T15:56:06,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:06,197 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/980315b15ccb4524bb7d97c68537275c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/980315b15ccb4524bb7d97c68537275c 2024-11-07T15:56:06,202 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/B of 817b58ba44d0cf51fa5fc01867ebb7a6 into 980315b15ccb4524bb7d97c68537275c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:06,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:06,202 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/B, priority=13, startTime=1730994965776; duration=0sec 2024-11-07T15:56:06,202 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/db78caa4e5644342b2d7d88301d3b699 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699 2024-11-07T15:56:06,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:06,202 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:56:06,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:06,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:06,203 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/C is initiating minor compaction (all files) 2024-11-07T15:56:06,204 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/C in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:06,204 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/566d91c99b624e91ae3c72d764642ab2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=36.0 K 2024-11-07T15:56:06,204 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 566d91c99b624e91ae3c72d764642ab2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1730994960120 2024-11-07T15:56:06,204 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fd824f7d58e48a5a7f00b262c7440d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1730994961299 2024-11-07T15:56:06,205 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb13ba6947242049d423dadb91a394f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:06,207 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/A of 817b58ba44d0cf51fa5fc01867ebb7a6 into db78caa4e5644342b2d7d88301d3b699(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:06,207 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:06,207 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/A, priority=13, startTime=1730994965776; duration=0sec 2024-11-07T15:56:06,207 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:06,207 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:56:06,211 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:06,211 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/fa5c6f20a0d546378d512bda7b192779 is 50, key is test_row_0/C:col10/1730994963462/Put/seqid=0 2024-11-07T15:56:06,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742390_1566 (size=12629) 2024-11-07T15:56:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:06,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995026296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995026297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995026299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995026299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995026413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995026413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995026413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995026413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,416 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:06,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110738728dfba0eb4c2ab783b56fae6b6a42_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738728dfba0eb4c2ab783b56fae6b6a42_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:06,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bd0caa1a79584f0b8bc89fef245d049c, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bd0caa1a79584f0b8bc89fef245d049c is 175, key is test_row_0/A:col10/1730994964137/Put/seqid=0 2024-11-07T15:56:06,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742391_1567 (size=31105) 2024-11-07T15:56:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:06,619 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/fa5c6f20a0d546378d512bda7b192779 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/fa5c6f20a0d546378d512bda7b192779 2024-11-07T15:56:06,623 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/C of 817b58ba44d0cf51fa5fc01867ebb7a6 into fa5c6f20a0d546378d512bda7b192779(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:06,623 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:06,623 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/C, priority=13, startTime=1730994965776; duration=0sec 2024-11-07T15:56:06,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:06,624 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:56:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995026623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995026623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995026623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995026623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bd0caa1a79584f0b8bc89fef245d049c 2024-11-07T15:56:06,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/32aea32531024d77ac05a2785c2a73b6 is 50, key is test_row_0/B:col10/1730994964137/Put/seqid=0 2024-11-07T15:56:06,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742392_1568 (size=12151) 2024-11-07T15:56:06,849 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/32aea32531024d77ac05a2785c2a73b6 2024-11-07T15:56:06,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4a142f3cbadf449e8c6821498b20f443 is 50, key is test_row_0/C:col10/1730994964137/Put/seqid=0 2024-11-07T15:56:06,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742393_1569 (size=12151) 2024-11-07T15:56:06,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995026933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995026933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995026934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:06,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995026934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:06,937 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:07,259 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4a142f3cbadf449e8c6821498b20f443 2024-11-07T15:56:07,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/bd0caa1a79584f0b8bc89fef245d049c as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c 2024-11-07T15:56:07,267 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c, entries=150, sequenceid=231, filesize=30.4 K 2024-11-07T15:56:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/32aea32531024d77ac05a2785c2a73b6 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6 2024-11-07T15:56:07,271 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6, entries=150, sequenceid=231, filesize=11.9 K 2024-11-07T15:56:07,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/4a142f3cbadf449e8c6821498b20f443 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443 2024-11-07T15:56:07,274 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443, entries=150, sequenceid=231, filesize=11.9 K 2024-11-07T15:56:07,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1257ms, sequenceid=231, compaction requested=false 2024-11-07T15:56:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:07,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-07T15:56:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-07T15:56:07,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-07T15:56:07,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-07T15:56:07,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4180 sec 2024-11-07T15:56:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:07,445 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:56:07,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-07T15:56:07,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fcc73df55afa4ca0a633544a2ac49524_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742394_1570 (size=12304) 2024-11-07T15:56:07,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995027463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995027467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995027468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995027468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995027573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995027583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995027583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995027583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995027783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,786 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995027793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,796 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995027793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995027793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:07,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:07,856 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:07,860 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107fcc73df55afa4ca0a633544a2ac49524_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fcc73df55afa4ca0a633544a2ac49524_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:07,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/a2f1bcdf941640d5b0229dfd668910a9, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:07,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/a2f1bcdf941640d5b0229dfd668910a9 is 175, key is test_row_0/A:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:07,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742395_1571 (size=31105) 2024-11-07T15:56:07,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/a2f1bcdf941640d5b0229dfd668910a9 2024-11-07T15:56:07,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/57654a7efe904a9fab059cff43e2f93a is 50, key is test_row_0/B:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:07,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742396_1572 (size=12151) 2024-11-07T15:56:07,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/57654a7efe904a9fab059cff43e2f93a 2024-11-07T15:56:07,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/0cc72f57d96340be939a191b33b9df49 is 50, key is test_row_0/C:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742397_1573 (size=12151) 2024-11-07T15:56:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-07T15:56:08,013 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:08,014 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-07T15:56:08,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-07T15:56:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:08,015 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-07T15:56:08,016 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-07T15:56:08,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-07T15:56:08,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995028092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995028103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995028103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995028103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:08,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:56:08,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:08,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T15:56:08,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/0cc72f57d96340be939a191b33b9df49 2024-11-07T15:56:08,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/a2f1bcdf941640d5b0229dfd668910a9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9 2024-11-07T15:56:08,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9, entries=150, sequenceid=253, filesize=30.4 K 2024-11-07T15:56:08,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/57654a7efe904a9fab059cff43e2f93a as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a 2024-11-07T15:56:08,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a, entries=150, sequenceid=253, filesize=11.9 K 2024-11-07T15:56:08,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/0cc72f57d96340be939a191b33b9df49 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49 2024-11-07T15:56:08,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49, entries=150, sequenceid=253, filesize=11.9 K 2024-11-07T15:56:08,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 855ms, sequenceid=253, compaction requested=true 2024-11-07T15:56:08,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:08,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:56:08,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:56:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:56:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93793 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/A is initiating minor compaction (all files) 2024-11-07T15:56:08,301 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/B is initiating minor compaction (all files) 2024-11-07T15:56:08,301 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/A in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,301 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/B in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,302 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/980315b15ccb4524bb7d97c68537275c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=36.1 K 2024-11-07T15:56:08,302 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=91.6 K 2024-11-07T15:56:08,302 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,302 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. files: [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9] 2024-11-07T15:56:08,302 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 980315b15ccb4524bb7d97c68537275c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:08,302 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting db78caa4e5644342b2d7d88301d3b699, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:08,302 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 32aea32531024d77ac05a2785c2a73b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730994964124 2024-11-07T15:56:08,302 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd0caa1a79584f0b8bc89fef245d049c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730994964124 2024-11-07T15:56:08,303 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 57654a7efe904a9fab059cff43e2f93a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730994966297 2024-11-07T15:56:08,303 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f1bcdf941640d5b0229dfd668910a9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730994966297 2024-11-07T15:56:08,308 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:08,309 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1e7e7088698e41eca19364f08480eb8d is 50, key is test_row_0/B:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:08,310 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:08,312 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241107bbd5bb2cfeba4afdbcd562bbf8e806ec_817b58ba44d0cf51fa5fc01867ebb7a6 store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:08,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742398_1574 (size=12731) 2024-11-07T15:56:08,314 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241107bbd5bb2cfeba4afdbcd562bbf8e806ec_817b58ba44d0cf51fa5fc01867ebb7a6, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:08,314 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107bbd5bb2cfeba4afdbcd562bbf8e806ec_817b58ba44d0cf51fa5fc01867ebb7a6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:08,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742399_1575 (size=4469) 2024-11-07T15:56:08,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42309 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-07T15:56:08,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-07T15:56:08,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110713299508711f4498b7f018602133c1ab_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994967460/Put/seqid=0 2024-11-07T15:56:08,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742400_1576 (size=12454) 2024-11-07T15:56:08,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:08,334 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110713299508711f4498b7f018602133c1ab_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110713299508711f4498b7f018602133c1ab_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/f215d5affe664fe8a678f647cd8f0da1, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:08,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/f215d5affe664fe8a678f647cd8f0da1 is 175, key is test_row_0/A:col10/1730994967460/Put/seqid=0 2024-11-07T15:56:08,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742401_1577 (size=31255) 2024-11-07T15:56:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:08,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. as already flushing 2024-11-07T15:56:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995028653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995028654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995028658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995028659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,717 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/1e7e7088698e41eca19364f08480eb8d as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1e7e7088698e41eca19364f08480eb8d 2024-11-07T15:56:08,718 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#A#compaction#478 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:08,718 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/9ecbc0dd91ab4c6aad4e6a28b8714160 is 175, key is test_row_0/A:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:08,721 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/B of 817b58ba44d0cf51fa5fc01867ebb7a6 into 1e7e7088698e41eca19364f08480eb8d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:08,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:08,722 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/B, priority=13, startTime=1730994968300; duration=0sec 2024-11-07T15:56:08,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:08,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:56:08,722 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-07T15:56:08,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742402_1578 (size=31685) 2024-11-07T15:56:08,723 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-07T15:56:08,723 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1541): 817b58ba44d0cf51fa5fc01867ebb7a6/C is initiating minor compaction (all files) 2024-11-07T15:56:08,723 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 817b58ba44d0cf51fa5fc01867ebb7a6/C in TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:08,723 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/fa5c6f20a0d546378d512bda7b192779, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49] into tmpdir=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp, totalSize=36.1 K 2024-11-07T15:56:08,723 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting fa5c6f20a0d546378d512bda7b192779, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1730994963462 2024-11-07T15:56:08,724 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a142f3cbadf449e8c6821498b20f443, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1730994964124 2024-11-07T15:56:08,724 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cc72f57d96340be939a191b33b9df49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1730994966297 2024-11-07T15:56:08,727 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/9ecbc0dd91ab4c6aad4e6a28b8714160 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/9ecbc0dd91ab4c6aad4e6a28b8714160 2024-11-07T15:56:08,731 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/A of 817b58ba44d0cf51fa5fc01867ebb7a6 into 9ecbc0dd91ab4c6aad4e6a28b8714160(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:08,731 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:08,731 INFO [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/A, priority=13, startTime=1730994968300; duration=0sec 2024-11-07T15:56:08,731 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:08,731 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:56:08,732 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 817b58ba44d0cf51fa5fc01867ebb7a6#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-07T15:56:08,732 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9d5a733658134c3fbc7929eab9364a9f is 50, key is test_row_0/C:col10/1730994966297/Put/seqid=0 2024-11-07T15:56:08,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742403_1579 (size=12731) 2024-11-07T15:56:08,739 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/f215d5affe664fe8a678f647cd8f0da1 2024-11-07T15:56:08,742 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9d5a733658134c3fbc7929eab9364a9f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9d5a733658134c3fbc7929eab9364a9f 2024-11-07T15:56:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/a7a4ae199b604f6fa0707169c2f82b38 is 50, key is test_row_0/B:col10/1730994967460/Put/seqid=0 2024-11-07T15:56:08,749 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 817b58ba44d0cf51fa5fc01867ebb7a6/C of 817b58ba44d0cf51fa5fc01867ebb7a6 into 9d5a733658134c3fbc7929eab9364a9f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-07T15:56:08,749 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:08,750 INFO [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., storeName=817b58ba44d0cf51fa5fc01867ebb7a6/C, priority=13, startTime=1730994968301; duration=0sec 2024-11-07T15:56:08,750 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:08,750 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:56:08,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742404_1580 (size=12301) 2024-11-07T15:56:08,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995028763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995028763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995028773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995028773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995028972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995028973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995028983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995028983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:08,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:09,071 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-07T15:56:09,071 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,071 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,071 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,072 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-07T15:56:09,072 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,072 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,072 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,072 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-07T15:56:09,072 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,072 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,073 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,073 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,075 INFO [Thread-2279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2279. 2024-11-07T15:56:09,075 DEBUG [Thread-2279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,075 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-07T15:56:09,075 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,075 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,075 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,075 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,075 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,075 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,153 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/a7a4ae199b604f6fa0707169c2f82b38 2024-11-07T15:56:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:09,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9152523629d74a8b93d0db1cbfbce16e is 50, key is test_row_0/C:col10/1730994967460/Put/seqid=0 2024-11-07T15:56:09,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742405_1581 (size=12301) 2024-11-07T15:56:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:09,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55198 deadline: 1730995029283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:09,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55182 deadline: 1730995029283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:09,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:09,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T15:56:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55214 deadline: 1730995029292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55200 deadline: 1730995029292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 , the old value is region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:09,293 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., hostname=c2682f91c1e2,42309,1730994796599, seqNum=5 because the exception is null or not the one we care about 2024-11-07T15:56:09,562 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9152523629d74a8b93d0db1cbfbce16e 2024-11-07T15:56:09,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/f215d5affe664fe8a678f647cd8f0da1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/f215d5affe664fe8a678f647cd8f0da1 2024-11-07T15:56:09,568 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/f215d5affe664fe8a678f647cd8f0da1, entries=150, sequenceid=271, filesize=30.5 K 2024-11-07T15:56:09,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/a7a4ae199b604f6fa0707169c2f82b38 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/a7a4ae199b604f6fa0707169c2f82b38 2024-11-07T15:56:09,572 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/a7a4ae199b604f6fa0707169c2f82b38, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:56:09,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/9152523629d74a8b93d0db1cbfbce16e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9152523629d74a8b93d0db1cbfbce16e 2024-11-07T15:56:09,575 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9152523629d74a8b93d0db1cbfbce16e, entries=150, sequenceid=271, filesize=12.0 K 2024-11-07T15:56:09,576 INFO [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1254ms, sequenceid=271, compaction requested=false 2024-11-07T15:56:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c2682f91c1e2:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-07T15:56:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-07T15:56:09,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-07T15:56:09,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-11-07T15:56:09,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.5650 sec 2024-11-07T15:56:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42309 {}] regionserver.HRegion(8855): Flush requested on 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:09,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-07T15:56:09,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-07T15:56:09,794 INFO [Thread-2270 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2270. 2024-11-07T15:56:09,795 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-07T15:56:09,795 DEBUG [Thread-2270 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,795 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,795 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,795 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,795 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,795 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710b990e5191c4da3a1564507b204e1c2_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994969793/Put/seqid=0 2024-11-07T15:56:09,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742406_1582 (size=12454) 2024-11-07T15:56:09,805 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-07T15:56:09,805 INFO [Thread-2268 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2268. 2024-11-07T15:56:09,805 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,805 DEBUG [Thread-2268 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:09,805 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,805 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,805 DEBUG [Thread-2268 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,805 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:09,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:09,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-07T15:56:10,163 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:10,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:10,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110710b990e5191c4da3a1564507b204e1c2_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710b990e5191c4da3a1564507b204e1c2_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:10,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/d5c27f87948949c3b673ae6dc14a38b7, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:10,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/d5c27f87948949c3b673ae6dc14a38b7 is 175, key is test_row_0/A:col10/1730994969793/Put/seqid=0 2024-11-07T15:56:10,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742407_1583 (size=31255) 2024-11-07T15:56:10,225 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-07T15:56:10,225 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-07T15:56:10,225 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:10,225 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:10,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3470 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3434 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3406 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3480 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3426 2024-11-07T15:56:10,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-07T15:56:10,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:56:10,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:56:10,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:10,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:10,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:10,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-07T15:56:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:10,229 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994970229"}]},"ts":"1730994970229"} 2024-11-07T15:56:10,230 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-07T15:56:10,230 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-07T15:56:10,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-07T15:56:10,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, UNASSIGN}] 2024-11-07T15:56:10,232 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, UNASSIGN 2024-11-07T15:56:10,233 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=CLOSING, regionLocation=c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:10,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, UNASSIGN because future has completed 2024-11-07T15:56:10,234 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-07T15:56:10,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599}] 2024-11-07T15:56:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:10,386 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:10,386 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-07T15:56:10,386 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing 817b58ba44d0cf51fa5fc01867ebb7a6, disabling compactions & flushes 2024-11-07T15:56:10,386 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:10,611 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/d5c27f87948949c3b673ae6dc14a38b7 2024-11-07T15:56:10,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/70cb6768a9b64fd6a028137b0abb5a48 is 50, key is test_row_0/B:col10/1730994969793/Put/seqid=0 2024-11-07T15:56:10,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742408_1584 (size=12301) 2024-11-07T15:56:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:11,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/70cb6768a9b64fd6a028137b0abb5a48 2024-11-07T15:56:11,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/7486052766654e6d82e1bef69ec9cb9f is 50, key is test_row_0/C:col10/1730994969793/Put/seqid=0 2024-11-07T15:56:11,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742409_1585 (size=12301) 2024-11-07T15:56:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:11,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/7486052766654e6d82e1bef69ec9cb9f 2024-11-07T15:56:11,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/d5c27f87948949c3b673ae6dc14a38b7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/d5c27f87948949c3b673ae6dc14a38b7 2024-11-07T15:56:11,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/d5c27f87948949c3b673ae6dc14a38b7, entries=150, sequenceid=294, filesize=30.5 K 2024-11-07T15:56:11,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/70cb6768a9b64fd6a028137b0abb5a48 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/70cb6768a9b64fd6a028137b0abb5a48 2024-11-07T15:56:11,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/70cb6768a9b64fd6a028137b0abb5a48, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T15:56:11,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/7486052766654e6d82e1bef69ec9cb9f as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/7486052766654e6d82e1bef69ec9cb9f 2024-11-07T15:56:11,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/7486052766654e6d82e1bef69ec9cb9f, entries=150, sequenceid=294, filesize=12.0 K 2024-11-07T15:56:11,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1647ms, sequenceid=294, compaction requested=true 2024-11-07T15:56:11,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 817b58ba44d0cf51fa5fc01867ebb7a6: 2024-11-07T15:56:11,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:A, priority=-2147483648, current under compaction store size is 1 2024-11-07T15:56:11,441 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:11,442 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:B, priority=-2147483648, current under compaction store size is 2 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. because compaction request was cancelled 2024-11-07T15:56:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-07T15:56:11,442 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. after waiting 0 ms 2024-11-07T15:56:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 817b58ba44d0cf51fa5fc01867ebb7a6:C, priority=-2147483648, current under compaction store size is 3 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. because compaction request was cancelled 2024-11-07T15:56:11,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:A 2024-11-07T15:56:11,442 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:B 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. because compaction request was cancelled 2024-11-07T15:56:11,442 DEBUG [RS:0;c2682f91c1e2:42309-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 817b58ba44d0cf51fa5fc01867ebb7a6:C 2024-11-07T15:56:11,442 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing 817b58ba44d0cf51fa5fc01867ebb7a6 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-07T15:56:11,446 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f3c8e52a54094d04a0ede70006cbceef_817b58ba44d0cf51fa5fc01867ebb7a6 is 50, key is test_row_0/A:col10/1730994969803/Put/seqid=0 2024-11-07T15:56:11,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742410_1586 (size=9914) 2024-11-07T15:56:11,849 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T15:56:11,852 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241107f3c8e52a54094d04a0ede70006cbceef_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f3c8e52a54094d04a0ede70006cbceef_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:11,853 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/98f1af7a3b4e46d1afbd0ac1ec568c23, store: [table=TestAcidGuarantees family=A region=817b58ba44d0cf51fa5fc01867ebb7a6] 2024-11-07T15:56:11,854 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/98f1af7a3b4e46d1afbd0ac1ec568c23 is 175, key is test_row_0/A:col10/1730994969803/Put/seqid=0 2024-11-07T15:56:11,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742411_1587 (size=22561) 2024-11-07T15:56:12,257 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/98f1af7a3b4e46d1afbd0ac1ec568c23 2024-11-07T15:56:12,263 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/40ee3802dfc048e9a781795b15899cc1 is 50, key is test_row_0/B:col10/1730994969803/Put/seqid=0 2024-11-07T15:56:12,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742412_1588 (size=9857) 2024-11-07T15:56:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:12,666 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/40ee3802dfc048e9a781795b15899cc1 2024-11-07T15:56:12,676 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/2734603dbe3b41c2ab4b5859884a47c9 is 50, key is test_row_0/C:col10/1730994969803/Put/seqid=0 2024-11-07T15:56:12,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742413_1589 (size=9857) 2024-11-07T15:56:13,079 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/2734603dbe3b41c2ab4b5859884a47c9 2024-11-07T15:56:13,083 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/A/98f1af7a3b4e46d1afbd0ac1ec568c23 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/98f1af7a3b4e46d1afbd0ac1ec568c23 2024-11-07T15:56:13,086 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/98f1af7a3b4e46d1afbd0ac1ec568c23, entries=100, sequenceid=300, filesize=22.0 K 2024-11-07T15:56:13,087 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/B/40ee3802dfc048e9a781795b15899cc1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/40ee3802dfc048e9a781795b15899cc1 2024-11-07T15:56:13,089 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/40ee3802dfc048e9a781795b15899cc1, entries=100, sequenceid=300, filesize=9.6 K 2024-11-07T15:56:13,090 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/.tmp/C/2734603dbe3b41c2ab4b5859884a47c9 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2734603dbe3b41c2ab4b5859884a47c9 2024-11-07T15:56:13,092 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2734603dbe3b41c2ab4b5859884a47c9, entries=100, sequenceid=300, filesize=9.6 K 2024-11-07T15:56:13,093 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1651ms, sequenceid=300, compaction requested=true 2024-11-07T15:56:13,094 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9] to archive 2024-11-07T15:56:13,095 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:56:13,096 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/54a4c800c01740fa9a201f497fa0181d 2024-11-07T15:56:13,097 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/3b8caaefd1ab4843bbe4bf4fc6a2b713 2024-11-07T15:56:13,098 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2ddb93b89d6d4f5e8e00bdf3f42e1334 2024-11-07T15:56:13,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/61c20ad198d64da39ecd756711f68a2a 2024-11-07T15:56:13,099 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/90e241358cc14ac8a9c7af1785f1b336 2024-11-07T15:56:13,100 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/fa4f19fff0584c349e336713ebba69e2 2024-11-07T15:56:13,101 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b126ad3836f6403392cdb51d786a5255 2024-11-07T15:56:13,102 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/cb51189eba9d4e9f8ff6fe039a3eedad 2024-11-07T15:56:13,103 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/b35b289fea79446385634bc1c680dfae 2024-11-07T15:56:13,104 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/978ea5d3e74e45e88bd33ae9ec1879c1 2024-11-07T15:56:13,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bff1a65691214afeb1db021f8c40a44f 2024-11-07T15:56:13,105 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/4e538ac39dac47f19c24c1bcb7ce6a9b 2024-11-07T15:56:13,106 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/2cc94be729d842fab4594c0588131e9e 2024-11-07T15:56:13,107 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/e2344d5595764fb18cba3e7f12a396a1 2024-11-07T15:56:13,108 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/db78caa4e5644342b2d7d88301d3b699 2024-11-07T15:56:13,109 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/bd0caa1a79584f0b8bc89fef245d049c 2024-11-07T15:56:13,110 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/a2f1bcdf941640d5b0229dfd668910a9 2024-11-07T15:56:13,111 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1baf5c37b0e64d3d9d70099c28facff0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ca9a8b2e2a314772885ad548cad592e5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/56a0fd8e01df43d7aabd77379c1a091f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/980315b15ccb4524bb7d97c68537275c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a] to archive 2024-11-07T15:56:13,112 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:56:13,113 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2f455ad389604466813de377d175815e 2024-11-07T15:56:13,114 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/fa13f63238b346fda2aa47c84c5812ff 2024-11-07T15:56:13,115 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1baf5c37b0e64d3d9d70099c28facff0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1baf5c37b0e64d3d9d70099c28facff0 2024-11-07T15:56:13,117 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/00f8483839b64e179ad8531f6f508939 2024-11-07T15:56:13,118 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bbfd82bbe2304bfb806546429ca9efd8 2024-11-07T15:56:13,119 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/476caa65a9714f688964a6b148234fc0 2024-11-07T15:56:13,120 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ca9a8b2e2a314772885ad548cad592e5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ca9a8b2e2a314772885ad548cad592e5 2024-11-07T15:56:13,121 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/bef2ad0ff2434865b9a1931616d161b8 2024-11-07T15:56:13,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1d7a807b420e49d98eb5906d38da91a7 2024-11-07T15:56:13,122 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/2c0741412ee3490d94e21b8d8d6df4e8 2024-11-07T15:56:13,123 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/56a0fd8e01df43d7aabd77379c1a091f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/56a0fd8e01df43d7aabd77379c1a091f 2024-11-07T15:56:13,124 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/74a5b3d1752647a19cb85c86bf9feff0 2024-11-07T15:56:13,125 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ec9e24ceff4f4f25ad127fa716851837 2024-11-07T15:56:13,126 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/980315b15ccb4524bb7d97c68537275c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/980315b15ccb4524bb7d97c68537275c 2024-11-07T15:56:13,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/ef2163df93334566822011050fcea0b5 2024-11-07T15:56:13,127 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/32aea32531024d77ac05a2785c2a73b6 2024-11-07T15:56:13,128 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/57654a7efe904a9fab059cff43e2f93a 2024-11-07T15:56:13,129 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/23d62d7cdb574495836d226cc615c4d5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2c859c9fea2d41f29ec9484d8276b41a, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/566d91c99b624e91ae3c72d764642ab2, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/fa5c6f20a0d546378d512bda7b192779, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49] to archive 2024-11-07T15:56:13,130 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-07T15:56:13,131 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/8e2ac3d276344b45b8ae7763780bcb21 2024-11-07T15:56:13,132 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/d6b46425b08a47baa322fc4f5f9a1af3 2024-11-07T15:56:13,133 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/23d62d7cdb574495836d226cc615c4d5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/23d62d7cdb574495836d226cc615c4d5 2024-11-07T15:56:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4cb591ad3f6c435db67ee4380ef6582d 2024-11-07T15:56:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/3740b0b8319a417f8f2748fdb5c94130 2024-11-07T15:56:13,135 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/b36f29e2bce84baaa65e1ec9ca729bae 2024-11-07T15:56:13,136 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2c859c9fea2d41f29ec9484d8276b41a to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2c859c9fea2d41f29ec9484d8276b41a 2024-11-07T15:56:13,138 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/50ad609a0f644151bc64eae7b9adfef5 2024-11-07T15:56:13,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/ea6ad55f6e0a4548ad59bab18872205b 2024-11-07T15:56:13,139 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/03ccd6f83b4a4d75aef075020fff863c 2024-11-07T15:56:13,140 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/566d91c99b624e91ae3c72d764642ab2 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/566d91c99b624e91ae3c72d764642ab2 2024-11-07T15:56:13,141 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/dfd4604b0a6145489132e24164a421c0 2024-11-07T15:56:13,142 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9fd824f7d58e48a5a7f00b262c7440d4 2024-11-07T15:56:13,143 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/fa5c6f20a0d546378d512bda7b192779 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/fa5c6f20a0d546378d512bda7b192779 2024-11-07T15:56:13,144 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/5fb13ba6947242049d423dadb91a394f 2024-11-07T15:56:13,145 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/4a142f3cbadf449e8c6821498b20f443 2024-11-07T15:56:13,146 DEBUG [StoreCloser-TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/0cc72f57d96340be939a191b33b9df49 2024-11-07T15:56:13,149 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-07T15:56:13,150 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. 2024-11-07T15:56:13,150 DEBUG [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for 817b58ba44d0cf51fa5fc01867ebb7a6: Waiting for close lock at 1730994970386Running coprocessor pre-close hooks at 1730994970386Disabling compacts and flushes for region at 1730994970386Disabling writes for close at 1730994971442 (+1056 ms)Obtaining lock to block concurrent updates at 1730994971442Preparing flush snapshotting stores in 817b58ba44d0cf51fa5fc01867ebb7a6 at 1730994971442Finished memstore snapshotting TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1730994971442Flushing stores of TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6. at 1730994971442Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/A: creating writer at 1730994971442Flushing mob file 817b58ba44d0cf51fa5fc01867ebb7a6/A: appending metadata at 1730994971446 (+4 ms)Flushing mob file 817b58ba44d0cf51fa5fc01867ebb7a6/A: closing flushed file at 1730994971446Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/A: appending metadata at 1730994971853 (+407 ms)Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/A: closing flushed file at 1730994971853Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/B: creating writer at 1730994972260 (+407 ms)Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/B: appending metadata at 1730994972262 (+2 ms)Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/B: closing flushed file at 1730994972262Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/C: creating writer at 1730994972674 (+412 ms)Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/C: appending metadata at 1730994972675 (+1 ms)Flushing 817b58ba44d0cf51fa5fc01867ebb7a6/C: closing flushed file at 1730994972676 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14192fe4: reopening flushed file at 1730994973083 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@400f4f3e: reopening flushed file at 1730994973086 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a3c93ce: reopening flushed file at 1730994973089 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 817b58ba44d0cf51fa5fc01867ebb7a6 in 1651ms, sequenceid=300, compaction requested=true at 1730994973093 (+4 ms)Writing region close event to WAL at 1730994973147 (+54 ms)Running coprocessor post-close hooks at 1730994973150 (+3 ms)Closed at 1730994973150 2024-11-07T15:56:13,152 INFO [RS_CLOSE_REGION-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed 817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:13,152 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=817b58ba44d0cf51fa5fc01867ebb7a6, regionState=CLOSED 2024-11-07T15:56:13,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 because future has completed 2024-11-07T15:56:13,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-07T15:56:13,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure 817b58ba44d0cf51fa5fc01867ebb7a6, server=c2682f91c1e2,42309,1730994796599 in 2.9200 sec 2024-11-07T15:56:13,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-07T15:56:13,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=817b58ba44d0cf51fa5fc01867ebb7a6, UNASSIGN in 2.9240 sec 2024-11-07T15:56:13,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-07T15:56:13,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9270 sec 2024-11-07T15:56:13,159 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730994973159"}]},"ts":"1730994973159"} 2024-11-07T15:56:13,160 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-07T15:56:13,160 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-07T15:56:13,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.9350 sec 2024-11-07T15:56:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-07T15:56:14,404 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:14,404 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-07T15:56:14,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:56:14,406 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,409 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,410 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C, FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits] 2024-11-07T15:56:14,412 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/98f1af7a3b4e46d1afbd0ac1ec568c23 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/98f1af7a3b4e46d1afbd0ac1ec568c23 2024-11-07T15:56:14,413 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/9ecbc0dd91ab4c6aad4e6a28b8714160 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/9ecbc0dd91ab4c6aad4e6a28b8714160 2024-11-07T15:56:14,414 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/d5c27f87948949c3b673ae6dc14a38b7 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/d5c27f87948949c3b673ae6dc14a38b7 2024-11-07T15:56:14,415 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/f215d5affe664fe8a678f647cd8f0da1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/A/f215d5affe664fe8a678f647cd8f0da1 2024-11-07T15:56:14,417 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1e7e7088698e41eca19364f08480eb8d to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/1e7e7088698e41eca19364f08480eb8d 2024-11-07T15:56:14,418 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/40ee3802dfc048e9a781795b15899cc1 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/40ee3802dfc048e9a781795b15899cc1 2024-11-07T15:56:14,419 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/70cb6768a9b64fd6a028137b0abb5a48 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/70cb6768a9b64fd6a028137b0abb5a48 2024-11-07T15:56:14,420 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/a7a4ae199b604f6fa0707169c2f82b38 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/B/a7a4ae199b604f6fa0707169c2f82b38 2024-11-07T15:56:14,421 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2734603dbe3b41c2ab4b5859884a47c9 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/2734603dbe3b41c2ab4b5859884a47c9 2024-11-07T15:56:14,422 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/7486052766654e6d82e1bef69ec9cb9f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/7486052766654e6d82e1bef69ec9cb9f 2024-11-07T15:56:14,423 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9152523629d74a8b93d0db1cbfbce16e to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9152523629d74a8b93d0db1cbfbce16e 2024-11-07T15:56:14,424 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9d5a733658134c3fbc7929eab9364a9f to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/C/9d5a733658134c3fbc7929eab9364a9f 2024-11-07T15:56:14,426 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits/303.seqid to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6/recovered.edits/303.seqid 2024-11-07T15:56:14,426 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/default/TestAcidGuarantees/817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,426 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-07T15:56:14,426 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:56:14,427 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-07T15:56:14,429 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710b990e5191c4da3a1564507b204e1c2_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110710b990e5191c4da3a1564507b204e1c2_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,430 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110713299508711f4498b7f018602133c1ab_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110713299508711f4498b7f018602133c1ab_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,430 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107139b79a941ca42888889ee142805a0b2_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107139b79a941ca42888889ee142805a0b2_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,431 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738728dfba0eb4c2ab783b56fae6b6a42_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110738728dfba0eb4c2ab783b56fae6b6a42_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,432 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107474cca62c59d45b6afd6cae37ac55937_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107474cca62c59d45b6afd6cae37ac55937_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,433 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83e992ce9f4d9c8060a0e04724c138_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411074d83e992ce9f4d9c8060a0e04724c138_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,434 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075b3f8723707e4e2cbf751b1d3892d1e9_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411075b3f8723707e4e2cbf751b1d3892d1e9_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,434 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076175338460f240f4866eeff9399b12c4_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076175338460f240f4866eeff9399b12c4_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,435 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110767244022c8be4b558c94a2db38d01e94_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110767244022c8be4b558c94a2db38d01e94_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,436 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a7c0dd5f000469b9fba6639d47e79fd_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411076a7c0dd5f000469b9fba6639d47e79fd_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,437 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079771f684fa9b4024ac7bc8a2b7ff9cc9_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411079771f684fa9b4024ac7bc8a2b7ff9cc9_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,438 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bcbf9d4edafa49b5b6cdafcdfa4375b7_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107bcbf9d4edafa49b5b6cdafcdfa4375b7_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,439 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfaa71f780a04bc4b98043f6f903a5ca_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107cfaa71f780a04bc4b98043f6f903a5ca_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,439 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f13d4281cf384892bc40d2ac015d3512_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f13d4281cf384892bc40d2ac015d3512_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,440 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f3c8e52a54094d04a0ede70006cbceef_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107f3c8e52a54094d04a0ede70006cbceef_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,441 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fcc73df55afa4ca0a633544a2ac49524_817b58ba44d0cf51fa5fc01867ebb7a6 to hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241107fcc73df55afa4ca0a633544a2ac49524_817b58ba44d0cf51fa5fc01867ebb7a6 2024-11-07T15:56:14,441 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-07T15:56:14,443 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,446 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-07T15:56:14,448 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-07T15:56:14,448 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,449 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-07T15:56:14,449 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1730994974449"}]},"ts":"9223372036854775807"} 2024-11-07T15:56:14,450 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-07T15:56:14,450 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 817b58ba44d0cf51fa5fc01867ebb7a6, NAME => 'TestAcidGuarantees,,1730994945951.817b58ba44d0cf51fa5fc01867ebb7a6.', STARTKEY => '', ENDKEY => ''}] 2024-11-07T15:56:14,450 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-07T15:56:14,450 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1730994974450"}]},"ts":"9223372036854775807"} 2024-11-07T15:56:14,452 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-07T15:56:14,452 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-07T15:56:14,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-07T15:56:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39943 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-07T15:56:14,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-07T15:56:14,514 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-07T15:56:14,523 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 237) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=329 (was 287) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6286 (was 6417) 2024-11-07T15:56:14,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T15:56:14,523 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:56:14,523 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:56:14,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:14,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:14,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T15:56:14,523 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T15:56:14,523 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1831818974, stopped=false 2024-11-07T15:56:14,524 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2682f91c1e2,39943,1730994795853 2024-11-07T15:56:14,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T15:56:14,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T15:56:14,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:56:14,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:56:14,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T15:56:14,526 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T15:56:14,526 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:56:14,526 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:14,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:56:14,527 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2682f91c1e2,42309,1730994796599' ***** 2024-11-07T15:56:14,527 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T15:56:14,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T15:56:14,528 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(959): stopping server c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2682f91c1e2:42309. 2024-11-07T15:56:14,528 DEBUG [RS:0;c2682f91c1e2:42309 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T15:56:14,528 DEBUG [RS:0;c2682f91c1e2:42309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T15:56:14,528 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T15:56:14,529 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T15:56:14,529 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T15:56:14,529 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-07T15:56:14,529 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:14,529 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T15:56:14,530 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T15:56:14,530 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T15:56:14,530 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T15:56:14,530 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T15:56:14,530 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-07T15:56:14,554 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/info/d2b0be08508f484fb2dbe8bf81cf06a3 is 95, key is TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97./info:/1730994830023/DeleteFamily/seqid=0 2024-11-07T15:56:14,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742414_1590 (size=6514) 2024-11-07T15:56:14,598 INFO [regionserver/c2682f91c1e2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T15:56:14,730 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:14,926 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T15:56:14,930 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:14,958 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/info/d2b0be08508f484fb2dbe8bf81cf06a3 2024-11-07T15:56:14,977 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/ns/d9c024b878b94a939680378b3d51426e is 93, key is TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97./ns:/1730994829983/DeleteFamily/seqid=0 2024-11-07T15:56:14,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742415_1591 (size=5906) 2024-11-07T15:56:15,130 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:15,202 INFO [regionserver/c2682f91c1e2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T15:56:15,202 INFO [regionserver/c2682f91c1e2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T15:56:15,330 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:15,381 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/ns/d9c024b878b94a939680378b3d51426e 2024-11-07T15:56:15,398 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/rep_barrier/415bb0a469474df6a80f78a4edcc70d7 is 102, key is TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97./rep_barrier:/1730994829983/DeleteFamily/seqid=0 2024-11-07T15:56:15,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742416_1592 (size=6025) 2024-11-07T15:56:15,530 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:15,731 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:15,802 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/rep_barrier/415bb0a469474df6a80f78a4edcc70d7 2024-11-07T15:56:15,820 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/table/cb7487e09c3a43baa2f4e04f27bc2b58 is 96, key is TestAcidGuarantees,,1730994799342.b2da43428956e53a96aa3524187a4a97./table:/1730994829983/DeleteFamily/seqid=0 2024-11-07T15:56:15,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742417_1593 (size=5936) 2024-11-07T15:56:15,931 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:16,131 DEBUG [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-07T15:56:16,223 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/table/cb7487e09c3a43baa2f4e04f27bc2b58 2024-11-07T15:56:16,227 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/info/d2b0be08508f484fb2dbe8bf81cf06a3 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/info/d2b0be08508f484fb2dbe8bf81cf06a3 2024-11-07T15:56:16,230 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/info/d2b0be08508f484fb2dbe8bf81cf06a3, entries=12, sequenceid=90, filesize=6.4 K 2024-11-07T15:56:16,230 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/ns/d9c024b878b94a939680378b3d51426e as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/ns/d9c024b878b94a939680378b3d51426e 2024-11-07T15:56:16,233 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/ns/d9c024b878b94a939680378b3d51426e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-07T15:56:16,233 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/rep_barrier/415bb0a469474df6a80f78a4edcc70d7 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/rep_barrier/415bb0a469474df6a80f78a4edcc70d7 2024-11-07T15:56:16,236 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/rep_barrier/415bb0a469474df6a80f78a4edcc70d7, entries=6, sequenceid=90, filesize=5.9 K 2024-11-07T15:56:16,236 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/.tmp/table/cb7487e09c3a43baa2f4e04f27bc2b58 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/table/cb7487e09c3a43baa2f4e04f27bc2b58 2024-11-07T15:56:16,238 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/table/cb7487e09c3a43baa2f4e04f27bc2b58, entries=7, sequenceid=90, filesize=5.8 K 2024-11-07T15:56:16,239 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false 2024-11-07T15:56:16,243 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-07T15:56:16,243 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:56:16,244 INFO [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T15:56:16,244 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730994974529Running coprocessor pre-close hooks at 1730994974529Disabling compacts and flushes for region at 1730994974529Disabling writes for close at 1730994974530 (+1 ms)Obtaining lock to block concurrent updates at 1730994974530Preparing flush snapshotting stores in 1588230740 at 1730994974530Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1730994974530Flushing stores of hbase:meta,,1.1588230740 at 1730994974530Flushing 1588230740/info: creating writer at 1730994974531 (+1 ms)Flushing 1588230740/info: appending metadata at 1730994974553 (+22 ms)Flushing 1588230740/info: closing flushed file at 1730994974553Flushing 1588230740/ns: creating writer at 1730994974962 (+409 ms)Flushing 1588230740/ns: appending metadata at 1730994974976 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1730994974977 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1730994975384 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1730994975398 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1730994975398Flushing 1588230740/table: creating writer at 1730994975805 (+407 ms)Flushing 1588230740/table: appending metadata at 1730994975820 (+15 ms)Flushing 1588230740/table: closing flushed file at 1730994975820Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@458d3ee2: reopening flushed file at 1730994976226 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@759efbb8: reopening flushed file at 1730994976230 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e33d07: reopening flushed file at 1730994976233 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3b86e816: reopening flushed file at 1730994976236 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1709ms, sequenceid=90, compaction requested=false at 1730994976239 (+3 ms)Writing region close event to WAL at 1730994976241 (+2 ms)Running coprocessor post-close hooks at 1730994976243 (+2 ms)Closed at 1730994976244 (+1 ms) 2024-11-07T15:56:16,244 DEBUG [RS_CLOSE_META-regionserver/c2682f91c1e2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T15:56:16,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T15:56:16,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T15:56:16,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-07T15:56:16,331 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(976): stopping server c2682f91c1e2,42309,1730994796599; all regions closed. 2024-11-07T15:56:16,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741834_1010 (size=25344) 2024-11-07T15:56:16,338 DEBUG [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/oldWALs 2024-11-07T15:56:16,338 INFO [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2682f91c1e2%2C42309%2C1730994796599.meta:.meta(num 1730994798953) 2024-11-07T15:56:16,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741833_1009 (size=13699748) 2024-11-07T15:56:16,341 DEBUG [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/oldWALs 2024-11-07T15:56:16,341 INFO [RS:0;c2682f91c1e2:42309 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2682f91c1e2%2C42309%2C1730994796599:(num 1730994798362) 2024-11-07T15:56:16,341 DEBUG [RS:0;c2682f91c1e2:42309 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T15:56:16,341 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T15:56:16,341 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T15:56:16,341 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.ChoreService(370): Chore service for: regionserver/c2682f91c1e2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T15:56:16,341 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T15:56:16,342 INFO [regionserver/c2682f91c1e2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T15:56:16,342 INFO [RS:0;c2682f91c1e2:42309 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42309 2024-11-07T15:56:16,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2682f91c1e2,42309,1730994796599 2024-11-07T15:56:16,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T15:56:16,346 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T15:56:16,348 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2682f91c1e2,42309,1730994796599] 2024-11-07T15:56:16,349 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2682f91c1e2,42309,1730994796599 already deleted, retry=false 2024-11-07T15:56:16,349 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2682f91c1e2,42309,1730994796599 expired; onlineServers=0 2024-11-07T15:56:16,350 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2682f91c1e2,39943,1730994795853' ***** 2024-11-07T15:56:16,350 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T15:56:16,350 INFO [M:0;c2682f91c1e2:39943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T15:56:16,350 INFO [M:0;c2682f91c1e2:39943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T15:56:16,350 DEBUG [M:0;c2682f91c1e2:39943 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T15:56:16,350 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T15:56:16,350 DEBUG [M:0;c2682f91c1e2:39943 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T15:56:16,350 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.large.0-1730994798090 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.large.0-1730994798090,5,FailOnTimeoutGroup] 2024-11-07T15:56:16,350 DEBUG [master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.small.0-1730994798091 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2682f91c1e2:0:becomeActiveMaster-HFileCleaner.small.0-1730994798091,5,FailOnTimeoutGroup] 2024-11-07T15:56:16,350 INFO [M:0;c2682f91c1e2:39943 {}] hbase.ChoreService(370): Chore service for: master/c2682f91c1e2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T15:56:16,350 INFO [M:0;c2682f91c1e2:39943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T15:56:16,350 DEBUG [M:0;c2682f91c1e2:39943 {}] master.HMaster(1795): Stopping service threads 2024-11-07T15:56:16,350 INFO [M:0;c2682f91c1e2:39943 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T15:56:16,351 INFO [M:0;c2682f91c1e2:39943 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T15:56:16,351 ERROR [M:0;c2682f91c1e2:39943 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35131 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35131,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-07T15:56:16,351 INFO [M:0;c2682f91c1e2:39943 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T15:56:16,351 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T15:56:16,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T15:56:16,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T15:56:16,352 DEBUG [M:0;c2682f91c1e2:39943 {}] zookeeper.ZKUtil(347): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T15:56:16,352 WARN [M:0;c2682f91c1e2:39943 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T15:56:16,353 INFO [M:0;c2682f91c1e2:39943 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/.lastflushedseqids 2024-11-07T15:56:16,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742418_1594 (size=72) 2024-11-07T15:56:16,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:56:16,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42309-0x101836fe7560001, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:56:16,449 INFO [RS:0;c2682f91c1e2:42309 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T15:56:16,449 INFO [RS:0;c2682f91c1e2:42309 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2682f91c1e2,42309,1730994796599; zookeeper connection closed. 2024-11-07T15:56:16,449 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e8ab3ad {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e8ab3ad 2024-11-07T15:56:16,450 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-07T15:56:16,762 INFO [M:0;c2682f91c1e2:39943 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T15:56:16,762 INFO [M:0;c2682f91c1e2:39943 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T15:56:16,763 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T15:56:16,763 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:56:16,763 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:56:16,763 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T15:56:16,763 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:56:16,763 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=736.18 KB heapSize=903.01 KB 2024-11-07T15:56:16,787 DEBUG [M:0;c2682f91c1e2:39943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0553963ba29644a99040974e3dbe4606 is 82, key is hbase:meta,,1/info:regioninfo/1730994799040/Put/seqid=0 2024-11-07T15:56:16,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742419_1595 (size=5672) 2024-11-07T15:56:17,191 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2052 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0553963ba29644a99040974e3dbe4606 2024-11-07T15:56:17,212 DEBUG [M:0;c2682f91c1e2:39943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75bf58cf28a34c80a89365d510279bd1 is 2275, key is \x00\x00\x00\x00\x00\x00\x00^/proc:d/1730994886994/Put/seqid=0 2024-11-07T15:56:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742420_1596 (size=43779) 2024-11-07T15:56:17,616 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=735.62 KB at sequenceid=2052 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75bf58cf28a34c80a89365d510279bd1 2024-11-07T15:56:17,620 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75bf58cf28a34c80a89365d510279bd1 2024-11-07T15:56:17,636 DEBUG [M:0;c2682f91c1e2:39943 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5040a0d5c6f04264af237309513e9914 is 69, key is c2682f91c1e2,42309,1730994796599/rs:state/1730994798115/Put/seqid=0 2024-11-07T15:56:17,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073742421_1597 (size=5156) 2024-11-07T15:56:18,040 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2052 (bloomFilter=true), to=hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5040a0d5c6f04264af237309513e9914 2024-11-07T15:56:18,043 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0553963ba29644a99040974e3dbe4606 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0553963ba29644a99040974e3dbe4606 2024-11-07T15:56:18,046 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0553963ba29644a99040974e3dbe4606, entries=8, sequenceid=2052, filesize=5.5 K 2024-11-07T15:56:18,047 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/75bf58cf28a34c80a89365d510279bd1 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/75bf58cf28a34c80a89365d510279bd1 2024-11-07T15:56:18,049 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 75bf58cf28a34c80a89365d510279bd1 2024-11-07T15:56:18,049 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/75bf58cf28a34c80a89365d510279bd1, entries=168, sequenceid=2052, filesize=42.8 K 2024-11-07T15:56:18,050 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5040a0d5c6f04264af237309513e9914 as hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5040a0d5c6f04264af237309513e9914 2024-11-07T15:56:18,052 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35131/user/jenkins/test-data/aa8a7818-91ce-d6f6-2c79-3e1b8907b059/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5040a0d5c6f04264af237309513e9914, entries=1, sequenceid=2052, filesize=5.0 K 2024-11-07T15:56:18,053 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(3140): Finished flush of dataSize ~736.18 KB/753845, heapSize ~902.71 KB/924376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2052, compaction requested=false 2024-11-07T15:56:18,054 INFO [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T15:56:18,054 DEBUG [M:0;c2682f91c1e2:39943 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730994976762Disabling compacts and flushes for region at 1730994976762Disabling writes for close at 1730994976763 (+1 ms)Obtaining lock to block concurrent updates at 1730994976763Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730994976763Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=753845, getHeapSize=924616, getOffHeapSize=0, getCellsCount=2245 at 1730994976763Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730994976765 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730994976765Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730994976787 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730994976787Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730994977195 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730994977212 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730994977212Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730994977620 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730994977636 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730994977636Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25373705: reopening flushed file at 1730994978043 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a12c847: reopening flushed file at 1730994978046 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ab550d: reopening flushed file at 1730994978049 (+3 ms)Finished flush of dataSize ~736.18 KB/753845, heapSize ~902.71 KB/924376, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1290ms, sequenceid=2052, compaction requested=false at 1730994978053 (+4 ms)Writing region close event to WAL at 1730994978054 (+1 ms)Closed at 1730994978054 2024-11-07T15:56:18,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45323 is added to blk_1073741830_1006 (size=888416) 2024-11-07T15:56:18,057 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T15:56:18,057 INFO [M:0;c2682f91c1e2:39943 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T15:56:18,057 INFO [M:0;c2682f91c1e2:39943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39943 2024-11-07T15:56:18,057 INFO [M:0;c2682f91c1e2:39943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T15:56:18,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:56:18,159 INFO [M:0;c2682f91c1e2:39943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T15:56:18,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39943-0x101836fe7560000, quorum=127.0.0.1:49390, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T15:56:18,165 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T15:56:18,168 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T15:56:18,168 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T15:56:18,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T15:56:18,168 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.log.dir/,STOPPED} 2024-11-07T15:56:18,171 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T15:56:18,171 WARN [BP-480826019-172.17.0.2-1730994792941 heartbeating to localhost/127.0.0.1:35131 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T15:56:18,171 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T15:56:18,171 WARN [BP-480826019-172.17.0.2-1730994792941 heartbeating to localhost/127.0.0.1:35131 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-480826019-172.17.0.2-1730994792941 (Datanode Uuid 4cd907ef-b693-4f91-9ddc-5a993d1d9072) service to localhost/127.0.0.1:35131 2024-11-07T15:56:18,173 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data1/current/BP-480826019-172.17.0.2-1730994792941 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T15:56:18,173 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/cluster_2374cf5d-deaa-9c39-b160-9b0b1b4624a9/data/data2/current/BP-480826019-172.17.0.2-1730994792941 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T15:56:18,174 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T15:56:18,180 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T15:56:18,181 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T15:56:18,181 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T15:56:18,181 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T15:56:18,181 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cf75675-3ac3-dace-527c-e7b523a3dd0a/hadoop.log.dir/,STOPPED} 2024-11-07T15:56:18,197 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T15:56:18,315 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down